diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..f49c19ef --- /dev/null +++ b/.gitattributes @@ -0,0 +1,5 @@ +*.sh text eol=lf +*.py text eol=lf +*.sql text eol=lf +*.yaml text eol=lf +Dockerfile text eol=lf diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..806a906a --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.12.8-slim-bookworm +WORKDIR /app +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt && useradd --uid 10001 --create-home app +COPY src/ src/ +COPY migrations/ migrations/ +COPY scripts/ scripts/ +COPY tests/ tests/ +COPY contracts/ contracts/ +COPY lab.json . +ENV PYTHONUNBUFFERED=1 PYTHONDONTWRITEBYTECODE=1 +USER app +EXPOSE 8080 +CMD ["gunicorn", "--bind", "0.0.0.0:8080", "--workers", "1", "--threads", "4", "--timeout", "30", "--graceful-timeout", "5", "src.app:app"] diff --git a/REPORT.md b/REPORT.md index c2679005..d1d51188 100644 --- a/REPORT.md +++ b/REPORT.md @@ -1,37 +1,68 @@ -# Отчёт — лабораторная 3 +# Лабораторная 3 — отчёт +Мартинес Анастасия, БПИ2303. Целевая оценка: **4**. Ветка solution/bpi2303-martines. -- Автор, группа: TODO -- Целевая оценка; выбранный трек 5, если нужен: TODO -- Commit/PR: TODO после публикации -- Использованные источники, библиотеки и помощь ИИ: TODO +## Запуск +Нужны Docker Compose v2 и Python 3.9+. Из корня репозитория: +```sh +python scripts/bootstrap.py +docker compose up -d --build --wait +python -X utf8 scripts/verify.py +``` +API через LB: http://127.0.0.1:8080. Только LB публикует порт. +Остановка: `docker compose down`; намеренное удаление всех учебных данных: +`docker compose down -v`. Secrets генерируются в игнорируемом secrets/, .env. +На Linux hooks restart.sh/stop-one.sh исполняемые; на Windows verify выбирает .cmd. +Команды make check/test/test-persistence/test-failover вызывают те же публичные +Python-проверки; BASE_URL=http://127.0.0.1:8080, ACTION_SCRIPT — соответствующий hook. +Полная OpenAPI validation опциональна: requirements-dev.txt + scripts/validate_openapi.py. -## Запуск из чистого clone +## Архитектура +Перенесены src, миграции, Dockerfile и bootstrap собственной lab2; skeleton lab3 +и публичные проверки сохранены. Python 3.12.8, PostgreSQL 16.6, Redis 7.4.1, +HAProxy 3.0.6, версии Python-библиотек закреплены requirements.txt. +Две API-службы api1/api2, без container_name и sticky session. Общие PostgreSQL +(данные, sessions, idempotency) и Redis (TTL=60, неизменяемые профили). +API по одному Gunicorn worker/4 threads; X-Instance-Id указывает службу. +Архитектура поддерживает отдельное масштабирование, конфигурация LB явно на две реплики. -TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных. +Идемпотентность: область user+chat+POST+key. Транзакционный advisory lock по +SHA-256 области сериализует одинаковые ключи. Сохранение сообщения и ответа +в одной транзакции. Повтор в течение 24 ч читает сохранённый JSON, иной текст — +409; сессия и доступ проверяются заново. Ошибки не сохраняются как результат. +Параметризованный SQL, индекс истории (chat_id,created_at,id), keyset без OFFSET. +Миграции — один сервис migrate до старта API, версии/checksum и advisory lock. +Rolling experiment заменяет ту же версию; несовместимые миграции требуют отдельного +expand/contract плана, не заявляется поддержка произвольной смены схемы без простоя. -## Архитектура и решения +HAProxy: readiness GET /health/ready каждую 1 с, fall=2/rise=1, timeout check/connect=1 с. +Ожидаемое обнаружение сбоя около 2–4 с, проверено восстановление менее 10 с. +Docker DNS resolver обновляет адреса после recreate, hold valid=1 с. +Нет автоматического retry POST на LB. Gunicorn SIGTERM прекращает приём и +даёт запросам 5 с, stop grace=8 с; принудительный тест stop -t2 допускает +оборванный ответ, который клиент повторяет с тем же ключом. +Пулы/SQL timeout и fallback Redis сохранены из lab2. ready зависит от PostgreSQL. +Лимиты API по 1 CPU/384 MiB, PostgreSQL 1 CPU/512 MiB, Redis и LB по 0.5 CPU/128 MiB. -TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab2, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации. +## Реальные результаты +19.11.2026, Windows/Docker Desktop, 16 логических CPU; environment/resources в evidence. +- smoke и persistence: PASS; StartedAt всех запущенных служб до/после в restart.json. +- публичный failover: 85 успешных записей, 5 переходных ошибок, восстановление 2.406 с. +- 24 конкурентных запроса через обе реплики: ровно одно сообщение. +- отказ api1: 89 записей, 8 ошибок, max success gap=1.125 с. +- отказ api2: 84 записи, 8 ошибок, max success gap=1.141 с. +- rolling recreate обеих API: 352 записи, 17 переходных ошибок, gap=1.156 с. +Во всех случаях подтверждённые сообщения сохранены и логические дубли отсутствуют. +- 30 с closed-loop POST, concurrency=4: 6542 успешных, 0 ошибок, 216.931 RPS; + p50=16 мс, p95=32 мс, p99=78 мс. CPU/RAM сняты во время нагрузки. +Сырые данные: evidence/{smoke,persistence,failover,replication}.txt, +replication.json, restart.json, load.json, resources.json. +Один повтор, без отдельного фиксированного прогрева; пулы уже прогреты functional tests. +Closed-loop и latency только успешных запросов не определяют production SLA. -## Доказательства критериев - -| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства | -| --- | --- | --- | --- | -| Публичный smoke | make test | TODO | TODO | -| Остальные пункты уровня 3 | TODO | TODO | TODO | -| Все пункты уровня 4, если заявлен | TODO | TODO | TODO | -| Выбранный трек 5, если заявлен | TODO | TODO | TODO | - -TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены. - -## Эксперименты и отказы - -TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения. - -| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных | -| --- | --- | --- | --- | -| TODO | TODO | TODO | TODO | - -## Ограничения и следующие шаги - -TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза. +## Ограничения и источники +Единственные LB/DB/host — SPOF; отказ всего узла не выдерживается. +Между страницами истории нет snapshot; поздний COMMIT с ранним timestamp может +оказаться позади cursor. Пароли Argon2id, сессии имеют expiry/logout; +HTTP допустим только на loopback с синтетическими данными. +Источники: контракт курса, Flask/Psycopg/Redis/HAProxy. Codex помог написать код, +тесты, провести измерения и оформить отчёт. Оценка 5 не заявляется. diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 00000000..f6610317 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,81 @@ +name: vp-lab3 +x-api: &api + build: . + image: vp-lab3-api + environment: &app-env + DATABASE_URL_FILE: /run/secrets/database_url + CACHE_URL: redis://redis:6379/0 + SESSION_TTL_SECONDS: ${SESSION_TTL_SECONDS:-3600} + secrets: [database_url] + depends_on: + migrate: + condition: service_completed_successfully + healthcheck: + test: [CMD, python, -c, "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/health/ready',timeout=4)"] + interval: 3s + timeout: 5s + retries: 20 + stop_grace_period: 8s + mem_limit: 384m + cpus: 1 +services: + api1: + <<: *api + environment: + <<: *app-env + INSTANCE_ID: api1 + api2: + <<: *api + environment: + <<: *app-env + INSTANCE_ID: api2 + migrate: + build: . + image: vp-lab3-api + command: [python, -m, src.db] + environment: + DATABASE_URL_FILE: /run/secrets/database_url + secrets: [database_url] + depends_on: + db: + condition: service_healthy + mem_limit: 128m + cpus: 0.5 + lb: + image: haproxy:3.0.6-alpine + ports: ["127.0.0.1:8080:8080"] + volumes: ["./infra/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg:ro"] + depends_on: + api1: + condition: service_healthy + api2: + condition: service_healthy + mem_limit: 128m + cpus: 0.5 + db: + image: postgres:16.6-bookworm + environment: + POSTGRES_USER: messenger + POSTGRES_DB: messenger + POSTGRES_PASSWORD_FILE: /run/secrets/db_password + secrets: [db_password] + volumes: [pgdata:/var/lib/postgresql/data] + healthcheck: + test: [CMD-SHELL, "pg_isready -U messenger -d messenger"] + interval: 3s + timeout: 3s + retries: 20 + mem_limit: 512m + cpus: 1 + redis: + image: redis:7.4.1-bookworm + command: [redis-server, --maxmemory, 64mb, --maxmemory-policy, allkeys-lru, --save, "", --appendonly, "no"] + mem_limit: 128m + cpus: 0.5 +volumes: + pgdata: +secrets: + db_password: + file: ./secrets/db_password + database_url: + file: ./secrets/database_url diff --git a/evidence/contract.txt b/evidence/contract.txt new file mode 100644 index 00000000..b62af9c5 --- /dev/null +++ b/evidence/contract.txt @@ -0,0 +1 @@ +PASS: offline skeleton structure, local refs and Python syntax; application NOT tested. diff --git a/evidence/environment.json b/evidence/environment.json new file mode 100644 index 00000000..39f1f691 --- /dev/null +++ b/evidence/environment.json @@ -0,0 +1,6 @@ +{ + "platform": "Windows-11-10.0.26200-SP0", + "cpu": "Intel64 Family 6 Model 186 Stepping 2, GenuineIntel", + "logical_cpus": 16, + "observed_at": "2026-11-19T08:26:42.137298+00:00" +} \ No newline at end of file diff --git a/evidence/failover.txt b/evidence/failover.txt new file mode 100644 index 00000000..9da97d08 --- /dev/null +++ b/evidence/failover.txt @@ -0,0 +1,3 @@ +Executing explicit failover hook; hook output suppressed to avoid leaking secrets. +{"observed_instances_before": 2, "successful_writes": 85, "failed_attempts": 5, "first_recovery_seconds": 2.406} +PASS: failover preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/last-stopped.json b/evidence/last-stopped.json new file mode 100644 index 00000000..80572051 --- /dev/null +++ b/evidence/last-stopped.json @@ -0,0 +1 @@ +{"service": "api1"} \ No newline at end of file diff --git a/evidence/load.json b/evidence/load.json new file mode 100644 index 00000000..e412d7ab --- /dev/null +++ b/evidence/load.json @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.157, + "concurrency": 4, + "successful_requests": 6542, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 216.931, + "successful_latency_ms": { + "p50": 16.0, + "p95": 32.0, + "p99": 78.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/load.txt b/evidence/load.txt new file mode 100644 index 00000000..e412d7ab --- /dev/null +++ b/evidence/load.txt @@ -0,0 +1,17 @@ +{ + "lab": 3, + "workload": "closed-loop POST messages; no automatic retries", + "requested_duration_seconds": 30.0, + "elapsed_seconds": 30.157, + "concurrency": 4, + "successful_requests": 6542, + "failed_requests": 0, + "error_ratio": 0.0, + "successful_rps": 216.931, + "successful_latency_ms": { + "p50": 16.0, + "p95": 32.0, + "p99": 78.0 + }, + "note": "Latency percentiles cover successes only. Timeouts/errors are counted separately; closed-loop has coordinated omission and client overhead." +} diff --git a/evidence/persistence.txt b/evidence/persistence.txt new file mode 100644 index 00000000..749fd29b --- /dev/null +++ b/evidence/persistence.txt @@ -0,0 +1,2 @@ +Executing explicit persistence hook; hook output suppressed to avoid leaking secrets. +PASS: persistence preserved confirmed data and existing session. Hook scope needs operator evidence. diff --git a/evidence/replication.json b/evidence/replication.json new file mode 100644 index 00000000..d7e74c12 --- /dev/null +++ b/evidence/replication.json @@ -0,0 +1,35 @@ +[ + { + "scenario": "concurrent_idempotency", + "requests": 24, + "replicas": [ + "api1", + "api2" + ], + "unique_messages": 1 + }, + { + "scenario": "stop_api1", + "duration_seconds": 6.297, + "writes": 89, + "errors": 8, + "rps": 14.134, + "max_success_gap_seconds": 1.125 + }, + { + "scenario": "stop_api2", + "duration_seconds": 6.406, + "writes": 84, + "errors": 8, + "rps": 13.113, + "max_success_gap_seconds": 1.141 + }, + { + "scenario": "rolling_recreation", + "duration_seconds": 20.157, + "writes": 352, + "errors": 17, + "rps": 17.463, + "max_success_gap_seconds": 1.156 + } +] \ No newline at end of file diff --git a/evidence/replication.txt b/evidence/replication.txt new file mode 100644 index 00000000..5e3ea35e --- /dev/null +++ b/evidence/replication.txt @@ -0,0 +1,61 @@ +[ + { + "scenario": "concurrent_idempotency", + "requests": 24, + "replicas": [ + "api1", + "api2" + ], + "unique_messages": 1 + }, + { + "scenario": "stop_api1", + "duration_seconds": 6.297, + "writes": 89, + "errors": 8, + "rps": 14.134, + "max_success_gap_seconds": 1.125 + }, + { + "scenario": "stop_api2", + "duration_seconds": 6.406, + "writes": 84, + "errors": 8, + "rps": 13.113, + "max_success_gap_seconds": 1.141 + }, + { + "scenario": "rolling_recreation", + "duration_seconds": 20.157, + "writes": 352, + "errors": 17, + "rps": 17.463, + "max_success_gap_seconds": 1.156 + } +] + Container vp-lab3-api1-1 Stopping + Container vp-lab3-api1-1 Stopped + Container vp-lab3-api1-1 Created + Container vp-lab3-api1-1 Starting + Container vp-lab3-api1-1 Started + Container vp-lab3-api1-1 Waiting + Container vp-lab3-api1-1 Healthy + Container vp-lab3-api2-1 Stopping + Container vp-lab3-api2-1 Stopped + Container vp-lab3-api2-1 Created + Container vp-lab3-api2-1 Starting + Container vp-lab3-api2-1 Started + Container vp-lab3-api2-1 Waiting + Container vp-lab3-api2-1 Healthy + Container vp-lab3-api1-1 Recreate + Container vp-lab3-api1-1 Recreated + Container vp-lab3-api1-1 Starting + Container vp-lab3-api1-1 Started + Container vp-lab3-api1-1 Waiting + Container vp-lab3-api1-1 Healthy + Container vp-lab3-api2-1 Recreate + Container vp-lab3-api2-1 Recreated + Container vp-lab3-api2-1 Starting + Container vp-lab3-api2-1 Started + Container vp-lab3-api2-1 Waiting + Container vp-lab3-api2-1 Healthy diff --git a/evidence/resources.json b/evidence/resources.json new file mode 100644 index 00000000..b49ee906 --- /dev/null +++ b/evidence/resources.json @@ -0,0 +1,30 @@ +[ + { + "at": "2026-11-19T08:26:11.185861+00:00", + "stats": "vp-lab3-api1-1 CPU=71.37% RAM=114MiB / 384MiB\nvp-lab3-api2-1 CPU=83.40% RAM=104.3MiB / 384MiB\nvp-lab3-db-1 CPU=42.70% RAM=39.61MiB / 512MiB\nvp-lab3-lb-1 CPU=23.50% RAM=13.06MiB / 128MiB\nvp-lab3-redis-1 CPU=0.57% RAM=8.145MiB / 128MiB" + }, + { + "at": "2026-11-19T08:26:15.940158+00:00", + "stats": "vp-lab3-api1-1 CPU=73.52% RAM=104.2MiB / 384MiB\nvp-lab3-api2-1 CPU=73.55% RAM=104.2MiB / 384MiB\nvp-lab3-db-1 CPU=40.67% RAM=40.7MiB / 512MiB\nvp-lab3-lb-1 CPU=26.07% RAM=13.25MiB / 128MiB\nvp-lab3-redis-1 CPU=0.44% RAM=8.145MiB / 128MiB" + }, + { + "at": "2026-11-19T08:26:20.721850+00:00", + "stats": "vp-lab3-api1-1 CPU=72.90% RAM=104.2MiB / 384MiB\nvp-lab3-api2-1 CPU=62.36% RAM=104.2MiB / 384MiB\nvp-lab3-db-1 CPU=40.12% RAM=41.57MiB / 512MiB\nvp-lab3-lb-1 CPU=28.37% RAM=13.84MiB / 128MiB\nvp-lab3-redis-1 CPU=0.30% RAM=8.27MiB / 128MiB" + }, + { + "at": "2026-11-19T08:26:25.439797+00:00", + "stats": "vp-lab3-api1-1 CPU=63.82% RAM=105.6MiB / 384MiB\nvp-lab3-api2-1 CPU=86.69% RAM=105.9MiB / 384MiB\nvp-lab3-db-1 CPU=36.57% RAM=42.3MiB / 512MiB\nvp-lab3-lb-1 CPU=27.73% RAM=14.45MiB / 128MiB\nvp-lab3-redis-1 CPU=0.30% RAM=8.145MiB / 128MiB" + }, + { + "at": "2026-11-19T08:26:30.206103+00:00", + "stats": "vp-lab3-api1-1 CPU=69.40% RAM=105.6MiB / 384MiB\nvp-lab3-api2-1 CPU=68.89% RAM=105.5MiB / 384MiB\nvp-lab3-db-1 CPU=44.58% RAM=43.32MiB / 512MiB\nvp-lab3-lb-1 CPU=28.67% RAM=14.51MiB / 128MiB\nvp-lab3-redis-1 CPU=0.31% RAM=8.145MiB / 128MiB" + }, + { + "at": "2026-11-19T08:26:34.922520+00:00", + "stats": "vp-lab3-api1-1 CPU=67.82% RAM=105.7MiB / 384MiB\nvp-lab3-api2-1 CPU=60.80% RAM=105.5MiB / 384MiB\nvp-lab3-db-1 CPU=51.38% RAM=44.7MiB / 512MiB\nvp-lab3-lb-1 CPU=24.44% RAM=14.71MiB / 128MiB\nvp-lab3-redis-1 CPU=0.45% RAM=8.145MiB / 128MiB" + }, + { + "at": "2026-11-19T08:26:39.660510+00:00", + "stats": "vp-lab3-api1-1 CPU=43.63% RAM=105.5MiB / 384MiB\nvp-lab3-api2-1 CPU=70.65% RAM=105.5MiB / 384MiB\nvp-lab3-db-1 CPU=30.53% RAM=45.79MiB / 512MiB\nvp-lab3-lb-1 CPU=20.64% RAM=14.63MiB / 128MiB\nvp-lab3-redis-1 CPU=0.33% RAM=8.145MiB / 128MiB" + } +] \ No newline at end of file diff --git a/evidence/restart.json b/evidence/restart.json new file mode 100644 index 00000000..ada9adf1 --- /dev/null +++ b/evidence/restart.json @@ -0,0 +1,66 @@ +{ + "before": [ + { + "name": "/vp-lab3-api1-1", + "id": "05fbc5e994b01d6a95d765f05e1adab9a2ebdbceec5b937dfc76fd76d90c8ee0", + "started_at": "2026-11-19T07:57:51.994317008Z", + "image": "vp-lab3-api" + }, + { + "name": "/vp-lab3-api2-1", + "id": "e07b74f91369c4bdf08b6a50761518c79eed4804750dc1494510d97c9819a2f9", + "started_at": "2026-11-19T07:57:52.051494732Z", + "image": "vp-lab3-api" + }, + { + "name": "/vp-lab3-db-1", + "id": "7eff663c0b92fb55ffbc87e0abf52c0b6cf9ed00848dc4c80f6c03c384af9fab", + "started_at": "2026-11-19T07:57:43.404724647Z", + "image": "postgres:16.6-bookworm" + }, + { + "name": "/vp-lab3-lb-1", + "id": "a6c594cd39311a245af596bff7555d6006e5dccad65e8156dcea621f5255074d", + "started_at": "2026-11-19T07:57:55.966296377Z", + "image": "haproxy:3.0.6-alpine" + }, + { + "name": "/vp-lab3-redis-1", + "id": "65e72fcdbc56d531d0a9e7b2d7ae712839b6cb9040193a63a5f3077bc82b0b65", + "started_at": "2026-11-19T07:57:43.415349754Z", + "image": "redis:7.4.1-bookworm" + } + ], + "after": [ + { + "name": "/vp-lab3-api1-1", + "id": "05fbc5e994b01d6a95d765f05e1adab9a2ebdbceec5b937dfc76fd76d90c8ee0", + "started_at": "2026-11-19T08:24:52.548786666Z", + "image": "vp-lab3-api" + }, + { + "name": "/vp-lab3-api2-1", + "id": "e07b74f91369c4bdf08b6a50761518c79eed4804750dc1494510d97c9819a2f9", + "started_at": "2026-11-19T08:24:52.757877287Z", + "image": "vp-lab3-api" + }, + { + "name": "/vp-lab3-db-1", + "id": "7eff663c0b92fb55ffbc87e0abf52c0b6cf9ed00848dc4c80f6c03c384af9fab", + "started_at": "2026-11-19T08:24:52.039125534Z", + "image": "postgres:16.6-bookworm" + }, + { + "name": "/vp-lab3-lb-1", + "id": "a6c594cd39311a245af596bff7555d6006e5dccad65e8156dcea621f5255074d", + "started_at": "2026-11-19T08:24:52.750598261Z", + "image": "haproxy:3.0.6-alpine" + }, + { + "name": "/vp-lab3-redis-1", + "id": "65e72fcdbc56d531d0a9e7b2d7ae712839b6cb9040193a63a5f3077bc82b0b65", + "started_at": "2026-11-19T08:24:52.755035623Z", + "image": "redis:7.4.1-bookworm" + } + ] +} \ No newline at end of file diff --git a/evidence/smoke.txt b/evidence/smoke.txt new file mode 100644 index 00000000..fc20f9dd --- /dev/null +++ b/evidence/smoke.txt @@ -0,0 +1 @@ +PASS: lab3 public API scenario; infrastructure and grading evidence remain separate. diff --git a/infra/haproxy.cfg b/infra/haproxy.cfg new file mode 100644 index 00000000..9e61009f --- /dev/null +++ b/infra/haproxy.cfg @@ -0,0 +1,27 @@ +global + maxconn 512 +defaults + mode http + timeout connect 1s + timeout client 30s + timeout server 30s + timeout check 1s + retries 0 +resolvers docker + nameserver dns 127.0.0.11:53 + resolve_retries 2 + timeout resolve 1s + timeout retry 1s + hold valid 1s + hold obsolete 1s + hold nx 1s +frontend ingress + bind :8080 + default_backend api +backend api + balance roundrobin + option httpchk GET /health/ready + http-check expect status 200 + default-server inter 1s fall 2 rise 1 resolvers docker init-addr libc,none + server api1 api1:8080 check + server api2 api2:8080 check diff --git a/migrations/001_initial.sql b/migrations/001_initial.sql new file mode 100644 index 00000000..093ce411 --- /dev/null +++ b/migrations/001_initial.sql @@ -0,0 +1,34 @@ +CREATE TABLE users ( + id uuid PRIMARY KEY, + username varchar(32) NOT NULL UNIQUE CHECK (username ~ '^[a-z0-9_]{3,32}$'), + display_name varchar(100) NOT NULL CHECK (length(trim(display_name)) > 0), + password_hash text NOT NULL, + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE chats ( + id uuid PRIMARY KEY, + title varchar(100) NOT NULL CHECK (length(trim(title)) > 0), + created_at timestamptz NOT NULL DEFAULT clock_timestamp() +); +CREATE TABLE members ( + chat_id uuid NOT NULL REFERENCES chats(id), + user_id uuid NOT NULL REFERENCES users(id), + PRIMARY KEY (chat_id, user_id) +); +CREATE INDEX members_user_chat ON members(user_id, chat_id); +CREATE INDEX chats_history ON chats(created_at, id); +CREATE TABLE messages ( + id uuid PRIMARY KEY, + chat_id uuid NOT NULL REFERENCES chats(id), + sender_id uuid NOT NULL REFERENCES users(id), + text varchar(4000) NOT NULL CHECK (length(trim(text)) > 0), + created_at timestamptz NOT NULL DEFAULT clock_timestamp(), + FOREIGN KEY (chat_id, sender_id) REFERENCES members(chat_id, user_id) +); +CREATE INDEX messages_history ON messages(chat_id, created_at, id); +CREATE TABLE sessions ( + token_hash char(64) PRIMARY KEY, + user_id uuid NOT NULL REFERENCES users(id), + expires_at timestamptz NOT NULL +); +CREATE INDEX sessions_expiry ON sessions(expires_at); diff --git a/migrations/002_idempotency.sql b/migrations/002_idempotency.sql new file mode 100644 index 00000000..f23ab309 --- /dev/null +++ b/migrations/002_idempotency.sql @@ -0,0 +1,10 @@ +CREATE TABLE idempotency ( + user_id uuid NOT NULL REFERENCES users(id), + chat_id uuid NOT NULL REFERENCES chats(id), + key varchar(128) NOT NULL, + request jsonb NOT NULL, + response jsonb NOT NULL, + expires_at timestamptz NOT NULL DEFAULT (clock_timestamp() + interval '24 hours'), + PRIMARY KEY (user_id,chat_id,key) +); +CREATE INDEX idempotency_expiry ON idempotency(expires_at); diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..474e8e80 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,19 @@ +Flask==3.1.0 +gunicorn==23.0.0 +psycopg[binary,pool]==3.2.3 +redis==5.2.1 +argon2-cffi==23.1.0 +# Transitive dependencies pinned to the verified Linux/Python 3.12 environment. +argon2-cffi-bindings==26.2.0 +blinker==1.9.0 +cffi==2.1.1 +click==8.5.1 +itsdangerous==2.2.0 +Jinja2==3.1.6 +MarkupSafe==3.0.4 +packaging==26.3 +psycopg-binary==3.2.3 +psycopg-pool==3.3.3 +pycparser==3.12 +typing_extensions==4.16.0 +Werkzeug==3.1.9 diff --git a/scripts/action.py b/scripts/action.py new file mode 100644 index 00000000..8c29f02e --- /dev/null +++ b/scripts/action.py @@ -0,0 +1,21 @@ +"""Explicit local Compose failure hooks. Never deletes volumes.""" +import json +import os +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +os.chdir(ROOT) +mode = sys.argv[1] +if mode == 'restart': + services = subprocess.check_output(['docker', 'compose', 'ps', '--services', '--status', 'running'], text=True).split() + subprocess.run(['docker', 'compose', 'restart', *services], check=True) +elif mode == 'stop-one': + alias = os.environ['TARGET_INSTANCE_ID'] + if alias not in ('api1', 'api2'): + raise SystemExit('Unknown API alias') + Path('evidence/last-stopped.json').write_text(json.dumps({'service': alias}), encoding='utf-8') + subprocess.run(['docker', 'compose', 'stop', '-t', '2', alias], check=True) +else: + raise SystemExit('Unknown action') diff --git a/scripts/bootstrap.py b/scripts/bootstrap.py new file mode 100644 index 00000000..e22aeaa7 --- /dev/null +++ b/scripts/bootstrap.py @@ -0,0 +1,21 @@ +"""Create local synthetic secrets once; never overwrite existing credentials.""" +import secrets +from pathlib import Path + +root = Path(__file__).resolve().parents[1] +directory = root / 'secrets' +directory.mkdir(exist_ok=True) +password_file = directory / 'db_password' +if not password_file.exists(): + with password_file.open('x', encoding='utf-8') as file: + file.write(secrets.token_urlsafe(32)) +password = password_file.read_text(encoding='utf-8').strip() +url_file = directory / 'database_url' +if not url_file.exists(): + with url_file.open('x', encoding='utf-8') as file: + file.write('postgresql://messenger:' + password + '@db:5432/messenger') +env = root / '.env' +if not env.exists(): + with env.open('x', encoding='utf-8') as file: + file.write('SESSION_TTL_SECONDS=3600\n') +print('Local configuration ready; credentials are not printed.') diff --git a/scripts/restart.cmd b/scripts/restart.cmd new file mode 100644 index 00000000..82d4fb96 --- /dev/null +++ b/scripts/restart.cmd @@ -0,0 +1,3 @@ +@echo off +cd /d "%~dp0.." +python scripts/action.py restart diff --git a/scripts/restart.sh b/scripts/restart.sh new file mode 100755 index 00000000..2651a499 --- /dev/null +++ b/scripts/restart.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/action.py restart diff --git a/scripts/stop-one.cmd b/scripts/stop-one.cmd new file mode 100644 index 00000000..97bf88df --- /dev/null +++ b/scripts/stop-one.cmd @@ -0,0 +1,3 @@ +@echo off +cd /d "%~dp0.." +python scripts/action.py stop-one diff --git a/scripts/stop-one.sh b/scripts/stop-one.sh new file mode 100755 index 00000000..2448345c --- /dev/null +++ b/scripts/stop-one.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -eu +cd "$(dirname "$0")/.." +exec python3 scripts/action.py stop-one diff --git a/scripts/verify.py b/scripts/verify.py new file mode 100644 index 00000000..a5b1dada --- /dev/null +++ b/scripts/verify.py @@ -0,0 +1,79 @@ +"""Run public and student acceptance, preserving sanitized evidence.""" +import json +import os +import platform +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +os.chdir(ROOT) +EVIDENCE = ROOT / 'evidence' +ENV = {**os.environ, 'PYTHONUTF8': '1', 'BASE_URL': os.getenv('BASE_URL', 'http://127.0.0.1:8080')} + + +def run(args, artifact): + result = subprocess.run(args, cwd=ROOT, capture_output=True, text=True, + encoding='utf-8', errors='replace', env=ENV) + output = result.stdout + result.stderr + (EVIDENCE / artifact).write_text(output, encoding='utf-8') + print(output, end='', flush=True) + if result.returncode: + raise SystemExit(result.returncode) + return output + + +def containers(): + ids = subprocess.check_output(['docker', 'compose', 'ps', '-q'], text=True).split() + raw = json.loads(subprocess.check_output(['docker', 'inspect', *ids], text=True)) + return [{'name': x['Name'], 'id': x['Id'], 'started_at': x['State']['StartedAt'], + 'image': x['Config']['Image']} for x in raw] + + +def main(): + run([sys.executable, 'scripts/check_contract.py'], 'contract.txt') + run([sys.executable, 'tests/smoke.py'], 'smoke.txt') + before = containers() + suffix = '.cmd' if os.name == 'nt' else '.sh' + run([sys.executable, 'tests/resilience.py', '--mode', 'persistence', + '--action-script', 'scripts/restart' + suffix], 'persistence.txt') + after = containers() + old = {x['id']: x for x in before} + assert all(x['id'] in old and x['started_at'] != old[x['id']]['started_at'] for x in after) + (EVIDENCE / 'restart.json').write_text(json.dumps({'before':before,'after':after},indent=2),encoding='utf-8') + try: + run([sys.executable, 'tests/resilience.py', '--mode', 'failover', + '--action-script', 'scripts/stop-one' + suffix], 'failover.txt') + finally: + subprocess.run(['docker','compose','up','-d','--no-deps','--wait','api1','api2'], check=True) + time.sleep(2) + run([sys.executable, 'student-tests/replication.py'], 'replication.txt') + if (ROOT / 'student-tests/attachments.py').exists(): + run([sys.executable, 'student-tests/attachments.py'], 'attachments.txt') + if (ROOT / 'student-tests/pipeline.py').exists(): + run([sys.executable, 'student-tests/pipeline.py'], 'pipeline.txt') + if (ROOT / 'student-tests/observability.py').exists(): + run([sys.executable, 'student-tests/observability.py'], 'observability.txt') + process = subprocess.Popen([sys.executable,'tests/load.py','--duration','30','--concurrency','4', + '--output','evidence/load.json'], stdout=subprocess.PIPE,stderr=subprocess.STDOUT, + text=True,encoding='utf-8',env=ENV) + ids = [x['id'] for x in containers()] + samples=[] + while process.poll() is None: + stats=subprocess.check_output(['docker','stats','--no-stream','--format', + '{{.Name}} CPU={{.CPUPerc}} RAM={{.MemUsage}}',*ids],text=True) + samples.append({'at':datetime.now(timezone.utc).isoformat(),'stats':stats.strip()}) + time.sleep(2) + output=process.communicate()[0] + (EVIDENCE/'load.txt').write_text(output,encoding='utf-8') + (EVIDENCE/'resources.json').write_text(json.dumps(samples,indent=2),encoding='utf-8') + assert process.returncode==0, output + (EVIDENCE/'environment.json').write_text(json.dumps({'platform':platform.platform(), + 'cpu':platform.processor(),'logical_cpus':os.cpu_count(),'observed_at':datetime.now(timezone.utc).isoformat()},indent=2),encoding='utf-8') + print(output,flush=True) + + +if __name__ == '__main__': + main() diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/app.py b/src/app.py new file mode 100644 index 00000000..682c984e --- /dev/null +++ b/src/app.py @@ -0,0 +1,390 @@ +"""Lab 2 API: PostgreSQL is authoritative; Redis caches immutable public profiles.""" +import base64 +import binascii +import hashlib +import json +import os +import re +import secrets +import uuid +from concurrent.futures import ThreadPoolExecutor, TimeoutError as FutureTimeout +from datetime import datetime +from threading import BoundedSemaphore + +import psycopg +import redis +from argon2 import PasswordHasher +from argon2.exceptions import VerificationError +from flask import Flask, g, jsonify, request +from psycopg_pool import PoolTimeout, TooManyRequests +from psycopg.types.json import Jsonb +from werkzeug.exceptions import HTTPException + +from src.db import make_pool, setting + +app = Flask(__name__) +app.config['MAX_CONTENT_LENGTH'] = 65536 +pool = make_pool() +cache = redis.Redis.from_url(setting('CACHE_URL'), socket_connect_timeout=0.2, + socket_timeout=0.2, decode_responses=True) +cache_executor = ThreadPoolExecutor(max_workers=2, thread_name_prefix='cache') +cache_slots = BoundedSemaphore(2) +hasher = PasswordHasher(time_cost=2, memory_cost=19456, parallelism=1) +dummy_hash = hasher.hash(secrets.token_urlsafe(32)) +SESSION_TTL = int(os.getenv('SESSION_TTL_SECONDS', '3600')) +if SESSION_TTL < 1: + raise ValueError('SESSION_TTL_SECONDS must be positive') +USER_FIELDS = 'id, username, display_name, created_at' +INSTANCE_ID = os.getenv('INSTANCE_ID', uuid.uuid4().hex) + + +class APIError(Exception): + def __init__(self, status, code, message, challenge=None): + self.status, self.code, self.message, self.challenge = status, code, message, challenge + + +def invalid(): + raise APIError(400, 'invalid_request', 'Invalid request') + + +def unauthorized(basic=False): + challenge = 'Basic realm="messenger", charset="UTF-8"' if basic else 'Bearer realm="messenger"' + raise APIError(401, 'unauthorized', 'Authentication required', challenge) + + +def serial(value): + if isinstance(value, dict): + return {key: serial(item) for key, item in value.items()} + if isinstance(value, list): + return [serial(item) for item in value] + if isinstance(value, uuid.UUID): + return str(value) + if isinstance(value, datetime): + return value.isoformat().replace('+00:00', 'Z') + return value + + +def response(value, status=200): + return jsonify(serial(value)), status + + +@app.before_request +def identify_request(): + g.request_id = uuid.uuid4().hex + + +@app.after_request +def headers(result): + result.headers['X-Request-Id'] = g.request_id + result.headers['X-Instance-Id'] = INSTANCE_ID + result.headers['Cache-Control'] = 'no-store' + return result + + +@app.errorhandler(Exception) +def error(exc): + if isinstance(exc, APIError): + problem = exc + elif isinstance(exc, psycopg.errors.UniqueViolation): + problem = APIError(409, 'conflict', 'Username already exists') + elif isinstance(exc, psycopg.errors.ForeignKeyViolation): + problem = APIError(400, 'invalid_request', 'Invalid member list') + elif isinstance(exc, (psycopg.OperationalError, PoolTimeout, TooManyRequests, + psycopg.errors.QueryCanceled, psycopg.errors.LockNotAvailable)): + problem = APIError(503, 'unavailable', 'Database temporarily unavailable') + elif isinstance(exc, HTTPException): + status = exc.code if exc.code in (400, 404, 413, 415) else 400 + code = {400: 'invalid_request', 404: 'not_found', 413: 'payload_too_large', + 415: 'unsupported_media_type'}[status] + problem = APIError(status, code, 'Request cannot be processed') + else: + app.logger.error('Unhandled error type=%s request_id=%s', type(exc).__name__, g.request_id) + problem = APIError(500, 'internal_error', 'Internal error') + result = jsonify(error={'code': problem.code, 'message': problem.message, + 'request_id': g.request_id}) + result.status_code = problem.status + if problem.challenge: + result.headers['WWW-Authenticate'] = problem.challenge + return result + + +def body(fields): + if request.mimetype != 'application/json': + raise APIError(415, 'unsupported_media_type', 'Expected application/json') + value = request.get_json() + if not isinstance(value, dict) or set(value) != set(fields): + invalid() + return value + + +def string(value, minimum, maximum): + if not isinstance(value, str) or not minimum <= len(value) <= maximum or not value.strip(): + invalid() + try: + value.encode('utf-8') + except UnicodeError: + invalid() + if '\x00' in value: + invalid() + return value + + +def uid(value): + try: + if not isinstance(value, str) or not re.fullmatch( + r'[0-9a-fA-F]{8}(?:-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}', value): + invalid() + return uuid.UUID(value) + except ValueError: + invalid() + + +def cached_operation(operation, *args, **kwargs): + # Socket timeouts do not bound DNS resolution when a Compose service is down. + # At most two background operations per worker, no unbounded queue. + if not cache_slots.acquire(blocking=False): + return None + future = cache_executor.submit(operation, *args, **kwargs) + future.add_done_callback(lambda _: cache_slots.release()) + try: + return future.result(timeout=0.25) + except (redis.RedisError, FutureTimeout): + return None + + +def profile(conn, user_id): + key = 'profile:v1:' + str(user_id) + cached = cached_operation(cache.get, key) + if cached: + try: + return json.loads(cached) + except ValueError: + pass + user = serial(conn.execute('SELECT ' + USER_FIELDS + ' FROM users WHERE id=%s', + (user_id,)).fetchone()) + if user is None: + unauthorized() + cached_operation(cache.set, key, json.dumps(user), ex=60) + return user + + +def actor(conn): + auth = request.headers.get('Authorization', '').split() + if len(auth) != 2 or auth[0].lower() != 'bearer' or len(auth[1]) > 256: + unauthorized() + digest = hashlib.sha256(auth[1].encode()).hexdigest() + session = conn.execute('SELECT user_id FROM sessions WHERE token_hash=%s ' + 'AND expires_at > clock_timestamp()', (digest,)).fetchone() + if not session: + unauthorized() + g.token_hash = digest + return session['user_id'] + + +def chat_access(conn, chat_id, user_id): + chat = conn.execute('SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id ' + 'WHERE c.id=%s AND m.user_id=%s', (chat_id, user_id)).fetchone() + if not chat: + raise APIError(404, 'not_found', 'Resource not found') + return chat + + +def chat_members(conn, chat): + chat['member_ids'] = [row['user_id'] for row in conn.execute( + 'SELECT user_id FROM members WHERE chat_id=%s ORDER BY user_id', (chat['id'],))] + return chat + + +def page_args(scope): + limit = request.args.get('limit', '50') + if not re.fullmatch(r'[0-9]{1,3}', limit) or not 1 <= int(limit) <= 100: + invalid() + cursor = request.args.get('cursor') + after = None + if cursor is not None: + try: + if len(cursor) > 1024: + invalid() + data = json.loads(base64.b64decode(cursor, altchars=b'-_', validate=True)) + if not isinstance(data, list) or len(data) != 3 or data[0] != scope: + invalid() + date = datetime.fromisoformat(data[1]) + if date.tzinfo is None: + invalid() + after = (date, uid(data[2])) + except (ValueError, TypeError, binascii.Error): + invalid() + return int(limit), after + + +def page(rows, limit, scope): + more = len(rows) > limit + items = rows[:limit] + cursor = None + if more: + last = serial(items[-1]) + cursor = base64.urlsafe_b64encode(json.dumps( + [scope, last['created_at'], last['id']]).encode()).decode() + return {'items': items, 'next_cursor': cursor} + + +@app.get('/health/live') +def live(): + return response({'status': 'ok'}) + + +@app.get('/health/ready') +def ready(): + with pool.connection() as conn: + conn.execute('SELECT 1 FROM schema_migrations LIMIT 1') + return response({'status': 'ok'}) + + +@app.post('/api/v1/users') +def register(): + data = body(('username', 'display_name', 'password')) + name = string(data['username'], 3, 32) + if not re.fullmatch('[a-z0-9_]{3,32}', name): + invalid() + display = string(data['display_name'], 1, 100) + password = string(data['password'], 12, 128) + encoded = hasher.hash(password) + with pool.connection() as conn: + user = conn.execute('INSERT INTO users (id,username,display_name,password_hash) ' + 'VALUES (%s,%s,%s,%s) RETURNING ' + USER_FIELDS, + (uuid.uuid4(), name, display, encoded)).fetchone() + return response(user, 201) + + +@app.post('/api/v1/auth/sessions') +def login(): + auth = request.headers.get('Authorization', '').split() + if len(auth) != 2 or auth[0].lower() != 'basic' or len(auth[1]) > 2048: + unauthorized(True) + try: + name, password = base64.b64decode(auth[1], validate=True).decode('utf-8').split(':', 1) + except (ValueError, UnicodeError, binascii.Error): + unauthorized(True) + if not re.fullmatch('[a-z0-9_]{3,32}', name) or not 12 <= len(password) <= 128: + unauthorized(True) + if request.get_data(): + invalid() + with pool.connection() as conn: + user = conn.execute('SELECT * FROM users WHERE username=%s', (name,)).fetchone() + try: + hasher.verify(user['password_hash'] if user else dummy_hash, password) + except VerificationError: + unauthorized(True) + if not user: + unauthorized(True) + token = secrets.token_urlsafe(32) + with pool.connection() as conn: + expiry = conn.execute('INSERT INTO sessions VALUES (%s,%s,clock_timestamp() ' + '+ %s * interval \'1 second\') RETURNING expires_at', + (hashlib.sha256(token.encode()).hexdigest(), user['id'], + SESSION_TTL)).fetchone()['expires_at'] + user.pop('password_hash') + return response({'token': token, 'token_type': 'Bearer', 'expires_at': expiry, 'user': user}, 201) + + +@app.delete('/api/v1/auth/sessions/current') +def logout(): + with pool.connection() as conn: + actor(conn) + conn.execute('DELETE FROM sessions WHERE token_hash=%s', (g.token_hash,)) + return '', 204 + + +@app.get('/api/v1/users/me') +def me(): + with pool.connection() as conn: + user = profile(conn, actor(conn)) + return response(user) + + +@app.post('/api/v1/chats') +def create_chat(): + with pool.connection() as conn: + user = actor(conn) + data = body(('title', 'member_ids')) + title = string(data['title'], 1, 100) + if not isinstance(data['member_ids'], list) or not 1 <= len(data['member_ids']) <= 100: + invalid() + members = [uid(item) for item in data['member_ids']] + if len(set(members)) != len(members) or user not in members: + invalid() + chat = conn.execute('INSERT INTO chats(id,title) VALUES (%s,%s) RETURNING *', + (uuid.uuid4(), title)).fetchone() + for member in members: + conn.execute('INSERT INTO members VALUES (%s,%s)', (chat['id'], member)) + chat['member_ids'] = sorted(members) + return response(chat, 201) + + +@app.get('/api/v1/chats') +def list_chats(): + with pool.connection() as conn: + user = actor(conn) + scope = 'chats:' + str(user) + limit, after = page_args(scope) + sql = 'SELECT c.* FROM chats c JOIN members m ON m.chat_id=c.id WHERE m.user_id=%s' + params = [user] + if after: + sql += ' AND (c.created_at,c.id) > (%s,%s)' + params.extend(after) + rows = conn.execute(sql + ' ORDER BY c.created_at,c.id LIMIT %s', params + [limit+1]).fetchall() + result = page(rows, limit, scope) + for chat in result['items']: + chat_members(conn, chat) + return response(result) + + +@app.get('/api/v1/chats/') +def get_chat(chat_id): + with pool.connection() as conn: + user = actor(conn) + result = chat_members(conn, chat_access(conn, uid(chat_id), user)) + return response(result) + + +@app.route('/api/v1/chats//messages', methods=['GET', 'POST']) +def messages(chat_id): + with pool.connection() as conn: + user = actor(conn) + chat_id = uid(chat_id) + chat_access(conn, chat_id, user) + if request.method == 'POST': + text = string(body(('text',))['text'], 1, 4000) + key = request.headers.get('Idempotency-Key', '') + if not re.fullmatch(r'[A-Za-z0-9._:-]{1,128}', key): + invalid() + canonical = {'text': text} + lock = int.from_bytes(hashlib.sha256(f'{user}:{chat_id}:{key}'.encode()).digest()[:8], + 'big', signed=True) + conn.execute('SELECT pg_advisory_xact_lock(%s)', (lock,)) + conn.execute('DELETE FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s ' + 'AND expires_at<=clock_timestamp()', (user, chat_id, key)) + prior = conn.execute('SELECT * FROM idempotency WHERE user_id=%s AND chat_id=%s AND key=%s', + (user, chat_id, key)).fetchone() + if prior: + if prior['request'] != canonical: + raise APIError(409, 'conflict', 'Idempotency key already used with a different message') + return response(prior['response'], 201) + result = conn.execute('INSERT INTO messages(id,chat_id,sender_id,text) ' + 'VALUES (%s,%s,%s,%s) RETURNING *', + (uuid.uuid4(), chat_id, user, text)).fetchone() + conn.execute('INSERT INTO idempotency(user_id,chat_id,key,request,response) ' + 'VALUES (%s,%s,%s,%s,%s)', + (user, chat_id, key, Jsonb(canonical), Jsonb(serial(result)))) + status = 201 + else: + scope = 'messages:' + str(chat_id) + limit, after = page_args(scope) + sql = 'SELECT * FROM messages WHERE chat_id=%s' + params = [chat_id] + if after: + sql += ' AND (created_at,id) > (%s,%s)' + params.extend(after) + rows = conn.execute(sql + ' ORDER BY created_at,id LIMIT %s', params + [limit+1]).fetchall() + result, status = page(rows, limit, scope), 200 + return response(result, status) diff --git a/src/db.py b/src/db.py new file mode 100644 index 00000000..19cf69cb --- /dev/null +++ b/src/db.py @@ -0,0 +1,52 @@ +"""Bounded PostgreSQL pool and serialized, versioned migrations.""" +import hashlib +import os +import time +from pathlib import Path + +import psycopg +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + + +def setting(name): + path = os.getenv(name + '_FILE') + return Path(path).read_text().strip() if path else os.environ[name] + + +def migrate(): + with psycopg.connect(setting('DATABASE_URL'), connect_timeout=3) as conn: + conn.execute('SELECT pg_advisory_xact_lock(202602)') + conn.execute('CREATE TABLE IF NOT EXISTS schema_migrations ' + '(version text PRIMARY KEY, checksum text NOT NULL)') + for file in sorted(Path('migrations').glob('*.sql')): + sql = file.read_text(encoding='utf-8') + digest = hashlib.sha256(sql.encode()).hexdigest() + row = conn.execute('SELECT checksum FROM schema_migrations WHERE version=%s', + (file.name,)).fetchone() + if row: + if row[0] != digest: + raise RuntimeError('Applied migration changed') + continue + conn.execute(sql) + conn.execute('INSERT INTO schema_migrations VALUES (%s,%s)', (file.name, digest)) + + +def make_pool(): + return ConnectionPool(setting('DATABASE_URL'), min_size=1, max_size=8, + timeout=3, max_waiting=32, + check=ConnectionPool.check_connection, + kwargs={'connect_timeout': 3, 'row_factory': dict_row, + 'options': '-c statement_timeout=3000 -c lock_timeout=2000'}) + + +if __name__ == '__main__': + for attempt in range(30): + try: + migrate() + break + except psycopg.OperationalError: + if attempt == 29: + raise RuntimeError('Database did not become available') from None + time.sleep(2) + print('Migrations applied successfully') diff --git a/student-tests/replication.py b/student-tests/replication.py new file mode 100644 index 00000000..99afac0e --- /dev/null +++ b/student-tests/replication.py @@ -0,0 +1,101 @@ +"""Concurrent idempotency, both replica failures, and rolling recreation under writes.""" +import json +import os +import subprocess +import sys +import threading +import time +import uuid +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from urllib.parse import quote + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / 'tests')) +from client import Client, check, Failure + + +def compose(*args): + subprocess.run(['docker', 'compose', *args], cwd=ROOT, check=True, + stdout=subprocess.DEVNULL, timeout=120) + + +def history(c, actor, chat): + path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100' + result = [] + while path: + page = c.call('GET', path, actor=actor, schema='MessagePage')['json'] + result.extend(page['items']) + path = '/api/v1/chats/' + chat['id'] + '/messages?limit=100&cursor=' + quote(page['next_cursor'], safe='') if page['next_cursor'] else None + return result + + +def main(): + c = Client(timeout=5) + alice, bob, eve, chat = c.bootstrap() + key = uuid.uuid4().hex + def send(_): + return Client().message(alice, chat, 'One logical message', key=key) + with ThreadPoolExecutor(max_workers=12) as executor: + responses = list(executor.map(send, range(24))) + ids = {item['json']['id'] for item in responses} + instances = {item['headers']['X-Instance-Id'] for item in responses} + check(len(ids) == 1 and len(instances) >= 2, 'Concurrency did not cover both replicas or duplicated message') + check(len(history(c, alice, chat)) == 1, 'Duplicate row committed') + c.call('POST', '/api/v1/chats/' + chat['id'] + '/messages', 409, actor=alice, + headers={'Idempotency-Key': key}, body={'text': 'Different'}) + results = [{'scenario': 'concurrent_idempotency', 'requests': 24, + 'replicas': sorted(instances), 'unique_messages': 1}] + + def experiment(label, action): + stop = threading.Event() + writes, errors, times = [], [], [] + def traffic(): + client = Client(timeout=3) + retry_key, text = uuid.uuid4().hex, label + uuid.uuid4().hex + while not stop.is_set(): + try: + response = client.message(alice, chat, text, key=retry_key) + writes.append(response['json']) + times.append(time.monotonic()) + retry_key, text = uuid.uuid4().hex, label + uuid.uuid4().hex + except Failure: + errors.append(time.monotonic()) + time.sleep(0.02) + thread = threading.Thread(target=traffic) + thread.start() + started = time.monotonic() + try: + action() + time.sleep(5) + finally: + stop.set() + thread.join(timeout=10) + duration = time.monotonic() - started + stored = {item['id']: item for item in history(c, bob, chat)} + check(writes and all(stored.get(item['id']) == item for item in writes), 'Confirmed writes lost') + bodies = [item['text'] for item in stored.values()] + check(len(bodies) == len(set(bodies)), 'Retry duplicated a logical message') + gaps = [b-a for a,b in zip([started]+times, times+[time.monotonic()])] + check(max(gaps) < 10, 'Recovery exceeded 10 seconds') + results.append({'scenario': label, 'duration_seconds': round(duration,3), + 'writes': len(writes), 'errors': len(errors), + 'rps': round(len(writes)/duration,3), 'max_success_gap_seconds': round(max(gaps),3)}) + + for service in ('api1', 'api2'): + try: + experiment('stop_' + service, lambda: compose('stop', '-t', '2', service)) + finally: + compose('up', '-d', '--no-deps', '--wait', service) + time.sleep(2) + def rolling(): + for service in ('api1', 'api2'): + compose('up', '-d', '--no-deps', '--force-recreate', '--wait', service) + time.sleep(2) + experiment('rolling_recreation', rolling) + Path(ROOT / 'evidence/replication.json').write_text(json.dumps(results, indent=2), encoding='utf-8') + print(json.dumps(results, indent=2)) + + +if __name__ == '__main__': + main()