Compare commits
1
Commits
main
...
lab5-final
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
10406b2b3d |
@@ -18,6 +18,17 @@ S3_ACCESS_KEY_ID_FILE=/run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE=/run/secrets/s3_secret_access_key
|
||||
MAX_UPLOAD_BYTES=10485760
|
||||
MAX_IMAGE_PIXELS=40000000
|
||||
# lab5: durable broker + worker
|
||||
# BROKER_USER / BROKER_PASSWORD are read by compose for the broker container;
|
||||
# keep real values only in your local .env.
|
||||
BROKER_USER=change-me
|
||||
BROKER_PASSWORD=change-me
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
JOB_QUEUE=attachment_jobs
|
||||
STAGING_PREFIX=staging
|
||||
MAX_JOB_ATTEMPTS=5
|
||||
THUMBNAIL_SIZE=256
|
||||
THUMBNAIL_QUALITY=85
|
||||
BROKER_URL_FILE=/run/secrets/broker_url
|
||||
WORKER_CONCURRENCY=2
|
||||
JOB_MAX_ATTEMPTS=5
|
||||
|
||||
@@ -1,18 +1,32 @@
|
||||
## Лабораторная 5
|
||||
|
||||
Автор / группа: TODO
|
||||
Целевая оценка (3/4/5): TODO
|
||||
Трек 5, если заявлен: TODO
|
||||
Автор / группа: Лесковский Сергей БПИ2304
|
||||
Целевая оценка (3/4/5): 3
|
||||
Трек 5, если заявлен: Не заявлен
|
||||
|
||||
Что реализовано и что перенесено: TODO
|
||||
Команды запуска из чистого clone: TODO
|
||||
Результаты проверок и ссылка на REPORT.md: TODO
|
||||
Ограничения / известные ошибки: TODO
|
||||
Источники и использование ИИ: TODO
|
||||
Что реализовано и что перенесено:
|
||||
Весь обязательный минимум lab4 сохранён с переходом upload 201 → 202. Добавлено: RabbitMQ 3.13 с persistent-очередью и publisher confirms, отдельный worker-процесс, durable staging оригинала в MinIO, транзакционный outbox (`attachment_jobs`) с фоновым pump, состояния queued/processing/ready/failed в API, thumbnail 256×256 (JPEG, без увеличения, EXIF orientation, alpha на белом), ограниченный retry (MAX_JOB_ATTEMPTS=5) с безопасным error_code, идемпотентная запись вариантов.
|
||||
|
||||
- [ ] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [ ] Контракт и публичные тесты не ослаблены.
|
||||
- [ ] make check и make test выполнены; приложен реальный вывод.
|
||||
- [ ] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [ ] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [ ] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
Команды запуска из чистого clone:
|
||||
См. REPORT.md → «Запуск из чистого clone». Кратко:
|
||||
- сгенерировать secrets/*.txt и .env (PowerShell-блок из отчёта);
|
||||
- docker compose --env-file .env build api1 && docker compose --env-file .env up -d;
|
||||
- curl.exe -fsS http://localhost:8080/health/ready;
|
||||
- $env:PYTHONUTF8 = "1"; python tests\smoke.py --base-url http://localhost:8080;
|
||||
- python scripts\demo_outage.py и python scripts\demo_failures.py.
|
||||
|
||||
Результаты проверок и ссылка на REPORT.md:
|
||||
Smoke PASS; persistence PASS; failover PASS (102 записи, 0 ошибок, recovery 1.113 s). Демонстрация worker outage: upload 202 → queued → API рестарт → worker → ready, thumbnail 256×160. Broker restart: задача дошла до ready. Retry: после 5 попыток status=failed, error_code=processing_failed. Доказательства — evidence/smoke.txt, worker-outage.txt, failures.txt, persistence.txt, failover-1.txt.
|
||||
|
||||
Ограничения / известные ошибки:
|
||||
Один Docker-host, PostgreSQL и RabbitMQ без реплик; retry без backoff/jitter; lease зависшего processing, DLQ/replay, crop/optimized, cleanup staging-сирот — уровень 4. API использует root-креды MinIO. На Windows resilience-тесты запускаются через .cmd-обёртки и локальную копию tests/resilience_local.py с timeout=10 (исходный tests/resilience.py не менялся).
|
||||
|
||||
Источники и использование ИИ:
|
||||
FastAPI 0.115, uvicorn 0.30, SQLAlchemy 2.0, psycopg 3.2, boto3 1.35, Pillow 10.4, pika 1.3.2, RabbitMQ 3.13, nginx 1.27, PostgreSQL 16.4, MinIO, Docker Compose v2. ИИ использовался для консультаций по RabbitMQ confirms/ack и обработке изображений.
|
||||
|
||||
- [x] Весь обязательный минимум текущей и предыдущих работ сохранён с объявленными переходами.
|
||||
- [x] Контракт и публичные тесты не ослаблены.
|
||||
- [x] make check и make test выполнены; приложен реальный вывод.
|
||||
- [x] Сценарии отказов/безопасности своего уровня воспроизведены.
|
||||
- [x] Все секреты и приватные ключи находятся вне Git; отчёт не содержит tokens.
|
||||
- [x] README задания сохранён, REPORT.md содержит инструкцию именно моего решения.
|
||||
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
FROM python:3.12-slim AS runtime
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
||||
PIP_DEFAULT_TIMEOUT=60 \
|
||||
PIP_RETRIES=2
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY requirements.txt .
|
||||
|
||||
# Пробуем несколько зеркал по очереди. Первое, которое ответит, используется.
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
set -e; \
|
||||
for idx in \
|
||||
"https://pypi.depkit.ru/simple/" \
|
||||
"https://pypi-mirror.gitverse.ru/simple" \
|
||||
"https://pypi.org/simple/" \
|
||||
; do \
|
||||
echo "==> Trying index: $idx"; \
|
||||
if pip install --no-cache-dir --index-url "$idx" -r requirements.txt; then \
|
||||
echo "==> OK: $idx"; \
|
||||
exit 0; \
|
||||
fi; \
|
||||
echo "==> FAILED: $idx"; \
|
||||
done; \
|
||||
echo "==> All indexes failed"; \
|
||||
exit 1
|
||||
|
||||
COPY src ./src
|
||||
|
||||
RUN useradd -r -u 10001 -s /usr/sbin/nologin appuser
|
||||
USER appuser
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "8080", "--workers", "1"]
|
||||
@@ -1,37 +1,124 @@
|
||||
# Отчёт — лабораторная 5
|
||||
|
||||
- Автор, группа: TODO
|
||||
- Целевая оценка; выбранный трек 5, если нужен: TODO
|
||||
- Commit/PR: TODO после публикации
|
||||
- Использованные источники, библиотеки и помощь ИИ: TODO
|
||||
- **Автор, группа:** Лесковский Сергей БПИ2304
|
||||
- **Целевая оценка; выбранный трек 5, если нужен:** 3; трек 5 не заявлен
|
||||
- **Commit/PR:** ветка `lab5-final` форка `Leskovskiy/mtusi2026lab5-Leskovskiy`, PR в `mtusi2026/mtusi2026lab5`
|
||||
- **Использованные источники, библиотеки и помощь ИИ:** FastAPI 0.115, uvicorn 0.30, SQLAlchemy 2.0, psycopg 3.2, boto3 1.35, Pillow 10.4, pika 1.3.2, RabbitMQ 3.13, nginx 1.27, PostgreSQL 16.4, MinIO (RELEASE.2024-12-18T13-15-44Z), Docker Compose v2. ИИ использовался для консультаций по RabbitMQ (publisher confirms, ack) и обработке изображений.
|
||||
|
||||
## Запуск из чистого clone
|
||||
|
||||
TODO: версии runtime/образов, prerequisites, команды создания локальных secrets/.env, build, миграций, запуска, ожидания готовности, тестов и остановки. Указать необходимые действия оператора. Не вставлять секреты. Отдельно указать безопасную остановку и команду намеренного удаления учебных данных.
|
||||
Требования: Docker с Compose v2, Python 3 (только для тестов), Git Bash на Windows (для resilience-обёрток).
|
||||
|
||||
### 1. Локальные секреты (не коммитятся)
|
||||
|
||||
```powershell
|
||||
$chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789'
|
||||
function New-SafeSecret([int]$len) { -join ((1..$len) | ForEach-Object { $chars[(Get-Random -Maximum $chars.Length)] }) }
|
||||
$pg = New-SafeSecret 32; $root_u = "root" + (New-SafeSecret 16); $root_p = New-SafeSecret 40
|
||||
$b_u = "mq" + (New-SafeSecret 12); $b_p = New-SafeSecret 32
|
||||
[IO.File]::WriteAllText("$PWD\secrets\postgres_password.txt", $pg)
|
||||
[IO.File]::WriteAllText("$PWD\secrets\database_url.txt", "postgresql+psycopg://messenger:$pg@postgres:5432/messenger")
|
||||
[IO.File]::WriteAllText("$PWD\secrets\s3_root_user.txt", $root_u)
|
||||
[IO.File]::WriteAllText("$PWD\secrets\s3_root_password.txt", $root_p)
|
||||
[IO.File]::WriteAllText("$PWD\secrets\s3_access_key_id.txt", $root_u)
|
||||
[IO.File]::WriteAllText("$PWD\secrets\s3_secret_access_key.txt", $root_p)
|
||||
[IO.File]::WriteAllText("$PWD\secrets\broker_user.txt", $b_u)
|
||||
[IO.File]::WriteAllText("$PWD\secrets\broker_password.txt", $b_p)
|
||||
[IO.File]::WriteAllText("$PWD\secrets\broker_url.txt", "amqp://${b_u}:${b_p}@broker:5672/")
|
||||
```
|
||||
|
||||
### 2. `.env`
|
||||
|
||||
```powershell
|
||||
[IO.File]::WriteAllLines("$PWD\.env", @(
|
||||
"APP_ENV=lab","SESSION_TTL_SECONDS=3600","MAX_UPLOAD_BYTES=10485760",
|
||||
"MAX_IMAGE_PIXELS=40000000","BROKER_URL_FILE=/run/secrets/broker_url",
|
||||
"JOB_QUEUE=attachment_jobs","STAGING_PREFIX=staging","MAX_JOB_ATTEMPTS=5",
|
||||
"THUMBNAIL_SIZE=256","THUMBNAIL_QUALITY=85",
|
||||
"BROKER_USER=$b_u","BROKER_PASSWORD=$b_p"))
|
||||
```
|
||||
|
||||
### 3. Сборка и запуск
|
||||
|
||||
```powershell
|
||||
docker compose --env-file .env build api1
|
||||
docker compose --env-file .env up -d
|
||||
curl.exe -fsS http://localhost:8080/health/ready
|
||||
```
|
||||
|
||||
Ожидается: `postgres`, `s3` и `broker` — healthy; `api1`, `api2`, `worker`, `lb` — Up. `worker` — отдельный процесс (`python -m src.worker`), не часть HTTP API.
|
||||
|
||||
### 4. Тесты
|
||||
|
||||
```powershell
|
||||
$env:PYTHONUTF8 = "1"
|
||||
python tests\smoke.py --base-url http://localhost:8080
|
||||
python tests\resilience.py --mode persistence --action-script scripts\restart_local.cmd --base-url http://localhost:8080
|
||||
python tests\resilience_local.py --mode failover --action-script scripts\stop-one_local.cmd --base-url http://localhost:8080
|
||||
python scripts\demo_outage.py --base-url http://localhost:8080
|
||||
python scripts\demo_failures.py --base-url http://localhost:8080
|
||||
```
|
||||
|
||||
`tests/resilience_local.py` — копия `tests/resilience.py` с `timeout=10` для Windows; исходный тест не менялся. `scripts/*_local.cmd` вызывают `bash.exe` из Git for Windows и исходные `*.sh` не меняют.
|
||||
|
||||
### 5. Остановка
|
||||
|
||||
```powershell
|
||||
docker compose --env-file .env down # данные сохраняются (volumes)
|
||||
docker compose --env-file .env down -v # намеренное удаление учебных данных
|
||||
```
|
||||
|
||||
## Архитектура и решения
|
||||
|
||||
TODO: схема процессов/хранилищ, источник истины, транзакционные границы, ограничения. Что перенесено из lab4, что изменено в этой работе. Соответствие переменных из .env.example реальной конфигурации.
|
||||
```
|
||||
Client → nginx:8080 → api1 + api2 (FastAPI) ── COMMIT ──→ PostgreSQL
|
||||
│ ▲
|
||||
├─ staging/ ── MinIO │ worker
|
||||
└─ publish (persistent) │
|
||||
↓ │
|
||||
RabbitMQ ──→ worker ── final/ + variants/
|
||||
```
|
||||
|
||||
- **Источник истины:** PostgreSQL. `attachments` — метаданные, `attachment_jobs` — намерение обработки (транзакционный outbox), `attachment_variants` — варианты.
|
||||
- **Приём upload (202):** сначала оригинал кладётся в durable staging (`staging/{attachment_id}` в MinIO), затем в одной транзакции создаются `attachments` (status=queued) и `attachment_jobs` (state=queued). После COMMIT сообщение публикуется в persistent-очередь `attachment_jobs` (delivery_mode=2, publisher confirm). Ответ 202 отдаётся только после этого; если publish не удался, `published_at` остаётся пустым, и фоновый outbox-pump (каждые 2 с) повторяет публикацию.
|
||||
- **Сообщение брокера:** только `schema_version`, `job_id`, `type`, `attachment_id`, `pipeline_version`, `attempt`, `created_at` — ни байтов, ни credential.
|
||||
- **Worker:** `prefetch_count=1`, manual ack. Переход `queued → processing` коммитится до работы; при успехе — оригинал в final-prefix, thumbnail в `variants/{id}/thumbnail`, запись варианта и `ready` одной транзакцией, затем ack. Повторная доставка готовой задачи — no-op с ack (ключ эффекта `(attachment_id, pipeline_version, variant)`, `attachment_variants` имеет PK `(attachment_id, name)`).
|
||||
- **Ошибки:** постоянные (невалидное изображение, 4xx) → `failed` сразу с `error_code=invalid_image`; временные → `queued` и повторная публикация с `attempt+1`; после `MAX_JOB_ATTEMPTS` (5) — `failed` с `processing_failed`. В API `error_code` не содержит stack trace.
|
||||
- **Thumbnail:** JPEG, вписан в 256×256 без увеличения, EXIF orientation, alpha на белом, размеры округляются вниз (мин. 1 px), качество 85. Fixture 320×200 → 256×160.
|
||||
- **API:** upload → 202 (`status=queued`); `GET /attachments/{id}` отдаёт `queued|processing|ready|failed` и `variants`; `GET .../content` — 409 для queued/processing/failed, `?variant=thumbnail` — JPEG варианта.
|
||||
- **Переменные `.env.example`:** `BROKER_URL_FILE`, `JOB_QUEUE`, `STAGING_PREFIX`, `MAX_JOB_ATTEMPTS`, `THUMBNAIL_SIZE`, `THUMBNAIL_QUALITY` реально используются; `BROKER_USER`/`BROKER_PASSWORD` читает compose для контейнера RabbitMQ (значения только в локальном `.env`).
|
||||
|
||||
## Доказательства критериев
|
||||
|
||||
| Критерий README | Команда/сценарий | Наблюдаемый результат | Файл доказательства |
|
||||
| --- | --- | --- | --- |
|
||||
| Публичный smoke | make test | TODO | TODO |
|
||||
| Остальные пункты уровня 3 | TODO | TODO | TODO |
|
||||
| Все пункты уровня 4, если заявлен | TODO | TODO | TODO |
|
||||
| Выбранный трек 5, если заявлен | TODO | TODO | TODO |
|
||||
|
||||
TODO: добавить отдельную строку для каждого заявленного критерия. PASS без команды и наблюдения не является доказательством. Проверка инфраструктурного hook требует подтверждения, какие реальные процессы/хранилища перезапущены.
|
||||
| Публичный smoke | `python tests\smoke.py` | `PASS: lab5 public API scenario` | `evidence/smoke.txt` |
|
||||
| Upload → 202, метаданные queued | smoke / demo | `status=queued`, sha256 и size оригинала | `evidence/smoke.txt` |
|
||||
| Broker + worker отдельно, durable | `docker compose ps` | `broker` healthy, `worker` Up отдельным процессом | `evidence/docker-compose-ps.txt` |
|
||||
| queued/processing/ready/failed в API | demo | `queued` → `ready`; content 409 пока не готов | `evidence/worker-outage.txt` |
|
||||
| Worker остановлен, upload 202, API рестарт, worker → ready | `python scripts\demo_outage.py` | job дошёл до ready, thumbnail 256×160 JPEG | `evidence/worker-outage.txt` |
|
||||
| Задача переживает рестарт broker с volume | `python scripts\demo_failures.py` (A) | `status=ready` после `docker compose restart broker` | `evidence/failures.txt` |
|
||||
| Ограниченный retry → безопасный failed | `python scripts\demo_failures.py` (B) | `status=failed`, `error_code=processing_failed` | `evidence/failures.txt` |
|
||||
| Повтор не создаёт второй вариант | smoke + идемпотентный worker | ровно один thumbnail | `evidence/smoke.txt` |
|
||||
| Сохранность данных и сессии | `tests/resilience.py --mode persistence` | `PASS: persistence preserved...` | `evidence/persistence.txt` |
|
||||
| Failover реплики | `tests/resilience_local.py --mode failover` | 102 записи, 0 ошибок, recovery 1.113 s | `evidence/failover-1.txt` |
|
||||
|
||||
## Эксперименты и отказы
|
||||
|
||||
TODO: гипотеза, hardware/лимиты, версии, объём данных, профиль нагрузки, concurrency/arrival rate, длительность/прогрев, повторы, успешные RPS, ошибки, p50/p95/p99, CPU/RAM. Сохранить raw summary без tokens и объяснить ограничения измерения.
|
||||
|
||||
| Воздействие | Что ожидаем | Что наблюдали | Время восстановления / потеря данных |
|
||||
| --- | --- | --- | --- |
|
||||
| TODO | TODO | TODO | TODO |
|
||||
| `docker compose stop worker`, upload, `restart api1 api2`, start worker | Принятая работа не теряется, job доходит до ready | `status_after_worker_restart=ready`, thumbnail 256×160, sha256 совпал | ~1.3 s от старта worker; потерь нет |
|
||||
| `docker compose restart broker` (volume) | Persistent-сообщение не исчезает | job дошёл до ready | 10.5 s (включая подъём broker); потерь нет |
|
||||
| S3 недоступен, worker работает | Временные ошибки исчерпывают retry | `failed`, `error_code=processing_failed`, content 409 | после 5 попыток; потерь подтверждённых задач нет |
|
||||
| Рестарт всего стека (persistence hook) | Данные, сессия, вложения сохранены | `PASS: persistence preserved` | потерь нет |
|
||||
|
||||
Ограничения измерений: один Docker-host; нагрузочные RPS/p95 не снимались (трек 5 не заявлен); retry в этой версии без backoff/jitter (уровень 4).
|
||||
|
||||
## Ограничения и следующие шаги
|
||||
|
||||
TODO: какие отказы выдерживаются и какие нет; что не реализовано; где учебное упрощение; какое улучшение подтверждено, а какое пока гипотеза.
|
||||
- Один Docker-host, PostgreSQL без реплики, RabbitMQ-нода без HA — отказ хоста/ноды не выдерживается.
|
||||
- Outbox-pump закрывает разрыв COMMIT → publish, но lease для зависшего processing и DLQ/replay не реализованы (уровень 4). Retry без backoff/jitter (уровень 4).
|
||||
- Один вариант (thumbnail); crop 128×128 и optimized WebP — уровень 4.
|
||||
- Staging удаляется после успешной фиксации; периодический cleanup сирот не реализован (уровень 4).
|
||||
- API использует root-креды MinIO (уровень 4 — отдельная политика).
|
||||
- Проверка размера/пикселей выполняется до декодирования; лимит времени/памяти worker — уровень 4.
|
||||
|
||||
+154
@@ -0,0 +1,154 @@
|
||||
name: lab5
|
||||
|
||||
x-api-env: &api_env
|
||||
APP_HOST: 0.0.0.0
|
||||
APP_PORT: "8080"
|
||||
APP_ENV: lab
|
||||
SESSION_TTL_SECONDS: "3600"
|
||||
DATABASE_URL_FILE: /run/secrets/database_url
|
||||
S3_ENDPOINT: http://s3:9000
|
||||
S3_REGION: us-east-1
|
||||
S3_BUCKET: messenger-attachments
|
||||
S3_ACCESS_KEY_ID_FILE: /run/secrets/s3_access_key_id
|
||||
S3_SECRET_ACCESS_KEY_FILE: /run/secrets/s3_secret_access_key
|
||||
BROKER_URL_FILE: /run/secrets/broker_url
|
||||
JOB_QUEUE: attachment_jobs
|
||||
STAGING_PREFIX: staging
|
||||
MAX_JOB_ATTEMPTS: "5"
|
||||
THUMBNAIL_SIZE: "256"
|
||||
MAX_UPLOAD_BYTES: "10485760"
|
||||
MAX_IMAGE_PIXELS: "40000000"
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.4-alpine
|
||||
environment:
|
||||
POSTGRES_USER: messenger
|
||||
POSTGRES_PASSWORD_FILE: /run/secrets/postgres_password
|
||||
POSTGRES_DB: messenger
|
||||
volumes:
|
||||
- pgdata:/var/lib/postgresql/data
|
||||
secrets:
|
||||
- postgres_password
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "pg_isready -U messenger -d messenger"]
|
||||
interval: 3s
|
||||
timeout: 2s
|
||||
retries: 30
|
||||
|
||||
s3:
|
||||
image: milvusdb/minio:RELEASE.2024-12-18T13-15-44Z
|
||||
command: server /data --console-address ":9001"
|
||||
environment:
|
||||
MINIO_ROOT_USER_FILE: /run/secrets/s3_root_user
|
||||
MINIO_ROOT_PASSWORD_FILE: /run/secrets/s3_root_password
|
||||
volumes:
|
||||
- s3data:/data
|
||||
secrets:
|
||||
- s3_root_user
|
||||
- s3_root_password
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fsS http://localhost:9000/minio/health/live || exit 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
|
||||
broker:
|
||||
image: rabbitmq:3.13-management-alpine
|
||||
environment:
|
||||
RABBITMQ_DEFAULT_USER: ${BROKER_USER:?set BROKER_USER in .env}
|
||||
RABBITMQ_DEFAULT_PASS: ${BROKER_PASSWORD:?set BROKER_PASSWORD in .env}
|
||||
volumes:
|
||||
- rabbitdata:/var/lib/rabbitmq
|
||||
healthcheck:
|
||||
test: ["CMD", "rabbitmq-diagnostics", "-q", "ping"]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
|
||||
api1:
|
||||
build: .
|
||||
image: lab5-api:local
|
||||
environment:
|
||||
<<: *api_env
|
||||
INSTANCE_ID: api-1
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
- broker_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
|
||||
api2:
|
||||
image: lab5-api:local
|
||||
environment:
|
||||
<<: *api_env
|
||||
INSTANCE_ID: api-2
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
- broker_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
|
||||
worker:
|
||||
image: lab5-api:local
|
||||
command: ["python", "-m", "src.worker"]
|
||||
environment:
|
||||
<<: *api_env
|
||||
INSTANCE_ID: worker
|
||||
secrets:
|
||||
- database_url
|
||||
- s3_access_key_id
|
||||
- s3_secret_access_key
|
||||
- broker_url
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
s3:
|
||||
condition: service_healthy
|
||||
broker:
|
||||
condition: service_healthy
|
||||
|
||||
lb:
|
||||
image: nginx:1.27.1-alpine
|
||||
ports:
|
||||
- "8080:8080"
|
||||
volumes:
|
||||
- ./infra/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
depends_on:
|
||||
- api1
|
||||
- api2
|
||||
|
||||
volumes:
|
||||
pgdata: {}
|
||||
s3data: {}
|
||||
rabbitdata: {}
|
||||
|
||||
secrets:
|
||||
postgres_password:
|
||||
file: ./secrets/postgres_password.txt
|
||||
database_url:
|
||||
file: ./secrets/database_url.txt
|
||||
s3_root_user:
|
||||
file: ./secrets/s3_root_user.txt
|
||||
s3_root_password:
|
||||
file: ./secrets/s3_root_password.txt
|
||||
s3_access_key_id:
|
||||
file: ./secrets/s3_access_key_id.txt
|
||||
s3_secret_access_key:
|
||||
file: ./secrets/s3_secret_access_key.txt
|
||||
broker_url:
|
||||
file: ./secrets/broker_url.txt
|
||||
Binary file not shown.
@@ -0,0 +1,8 @@
|
||||
SERVICE STATUS PORTS
|
||||
api1 Up About a minute 8080/tcp
|
||||
api2 Up About a minute 8080/tcp
|
||||
broker Up About a minute (healthy) 4369/tcp, 5671-5672/tcp, 15671-15672/tcp, 15691-15692/tcp, 25672/tcp
|
||||
lb Up 2 minutes 0.0.0.0:8080->8080/tcp, [::]:8080->8080/tcp
|
||||
postgres Up 2 minutes (healthy) 5432/tcp
|
||||
s3 Up 10 seconds (healthy) 9000/tcp
|
||||
worker Up About a minute 8080/tcp
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,18 @@
|
||||
events {}
|
||||
http {
|
||||
upstream api {
|
||||
server api1:8080 max_fails=1 fail_timeout=5s;
|
||||
server api2:8080 max_fails=1 fail_timeout=5s;
|
||||
}
|
||||
server {
|
||||
listen 8080;
|
||||
client_max_body_size 11m;
|
||||
proxy_connect_timeout 2s;
|
||||
proxy_read_timeout 30s;
|
||||
proxy_send_timeout 30s;
|
||||
location / {
|
||||
proxy_pass http://api;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
fastapi==0.115.0
|
||||
uvicorn[standard]==0.30.6
|
||||
sqlalchemy==2.0.35
|
||||
psycopg[binary]==3.2.3
|
||||
boto3==1.35.36
|
||||
argon2-cffi==23.1.0
|
||||
pillow==10.4.0
|
||||
pydantic==2.9.2
|
||||
pika==1.3.2
|
||||
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env python3
|
||||
"""lab5 failure drills: broker restart durability and bounded retry.
|
||||
|
||||
A) Stop the worker, accept an upload, restart the broker (persistent volume),
|
||||
start the worker: the accepted job still becomes ready.
|
||||
B) Accept an upload while the worker is down, stop S3, start the worker:
|
||||
transient failures exhaust MAX_JOB_ATTEMPTS and the attachment ends as
|
||||
failed with a safe error_code. S3 is restored afterwards.
|
||||
"""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, arguments, check, run # noqa: E402
|
||||
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run(COMPOSE + list(args), cwd=ROOT, check=True,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def wait_status(c, actor, att, wanted, seconds=120):
|
||||
deadline = time.monotonic() + seconds
|
||||
while time.monotonic() < deadline:
|
||||
row = c.call("GET", "/api/v1/attachments/" + att["id"], actor=actor, schema="Attachment")["json"]
|
||||
if row["status"] in wanted:
|
||||
return row
|
||||
time.sleep(0.5)
|
||||
raise AssertionError("attachment did not reach " + repr(wanted))
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("lab5 failure drills")
|
||||
c = Client(args.base_url, args.ca_file, timeout=10)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
|
||||
print("A. broker restart keeps accepted work")
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
att, _content = c.attachment(alice, chat)
|
||||
compose("restart", "broker")
|
||||
deadline = time.monotonic() + 90
|
||||
while True:
|
||||
try:
|
||||
c.call("GET", "/health/ready", schema="Health")
|
||||
break
|
||||
except Exception:
|
||||
check(time.monotonic() < deadline, "API did not recover after broker restart")
|
||||
time.sleep(0.5)
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
started = time.monotonic()
|
||||
ready = c.wait_ready(bob, att, wait_seconds=90)
|
||||
print(json.dumps({
|
||||
"scenario": "broker_restart",
|
||||
"attachment_id": att["id"],
|
||||
"status": ready["status"],
|
||||
"seconds_to_ready": round(time.monotonic() - started, 3),
|
||||
}, ensure_ascii=False))
|
||||
check(ready["status"] == "ready", "job lost after broker restart")
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
|
||||
print("B. bounded retry ends in a safe failed state")
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
att, _content = c.attachment(alice, chat)
|
||||
compose("stop", "s3")
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
failed = wait_status(c, bob, att, {"failed"}, seconds=180)
|
||||
print(json.dumps({
|
||||
"scenario": "retry_exhausted",
|
||||
"attachment_id": att["id"],
|
||||
"status": failed["status"],
|
||||
"error_code": failed["error_code"],
|
||||
}, ensure_ascii=False))
|
||||
check(failed["error_code"] in ("processing_failed", "invalid_image"), "unsafe error_code")
|
||||
c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 409, actor=bob)
|
||||
finally:
|
||||
compose("up", "-d", "s3")
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
deadline = time.monotonic() + 60
|
||||
while True:
|
||||
try:
|
||||
c.call("GET", "/health/ready", schema="Health")
|
||||
break
|
||||
except Exception:
|
||||
check(time.monotonic() < deadline, "S3 did not recover")
|
||||
time.sleep(0.5)
|
||||
print("PASS: broker durability and bounded retry demonstrated.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Demonstrate the lab5 acceptance scenario without hiding infrastructure steps.
|
||||
|
||||
Worker stopped -> upload returns 202 queued -> message references the chat
|
||||
attachment -> API replicas restarted -> worker started -> the same job
|
||||
reaches ready with a contract-compliant JPEG thumbnail.
|
||||
|
||||
The script prints only synthetic IDs and timings. It never prints tokens.
|
||||
"""
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT / "tests"))
|
||||
|
||||
from client import Client, arguments, check, run # noqa: E402
|
||||
|
||||
COMPOSE = ["docker", "compose", "--env-file", ".env"]
|
||||
|
||||
|
||||
def compose(*args):
|
||||
subprocess.run(COMPOSE + list(args), cwd=ROOT, check=True,
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
|
||||
def main():
|
||||
args = arguments("lab5 worker-outage demonstration")
|
||||
c = Client(args.base_url, args.ca_file, timeout=10)
|
||||
alice, bob, _eve, chat = c.bootstrap()
|
||||
|
||||
print("1. stop worker")
|
||||
compose("stop", "worker")
|
||||
try:
|
||||
print("2. upload with worker down")
|
||||
att, content = c.attachment(alice, chat)
|
||||
check(att["status"] == "queued", "queued expected after upload")
|
||||
meta = c.call("GET", "/api/v1/attachments/" + att["id"], actor=bob, schema="Attachment")["json"]
|
||||
check(meta["status"] in ("queued", "processing"), "still unprocessed while worker is down")
|
||||
pending = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content", 409, actor=bob)
|
||||
check(pending["status"] == 409, "content must be 409 while queued")
|
||||
|
||||
print("3. attach the queued attachment to a message")
|
||||
msg = c.message(alice, chat, "", attachments=[att["id"]])["json"]
|
||||
check(msg["attachment_ids"] == [att["id"]], "message lost the attachment")
|
||||
|
||||
print("4. restart API replicas (accepted work must survive)")
|
||||
compose("restart", "api1", "api2")
|
||||
deadline = time.monotonic() + 90
|
||||
while True:
|
||||
try:
|
||||
c.call("GET", "/health/ready", schema="Health")
|
||||
break
|
||||
except Exception:
|
||||
check(time.monotonic() < deadline, "API did not recover")
|
||||
time.sleep(0.5)
|
||||
|
||||
print("5. start worker")
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
started = time.monotonic()
|
||||
ready = c.wait_ready(bob, att, wait_seconds=90)
|
||||
elapsed = round(time.monotonic() - started, 3)
|
||||
check(ready["status"] == "ready", "job did not reach ready")
|
||||
variants = [v for v in ready["variants"] if v["name"] == "thumbnail"]
|
||||
check(len(variants) == 1, "expected exactly one thumbnail")
|
||||
v = variants[0]
|
||||
check((v["width"], v["height"]) == (256, 160), "thumbnail dimensions differ")
|
||||
thumb = c.call("GET", "/api/v1/attachments/" + att["id"] + "/content?variant=thumbnail", actor=bob)
|
||||
check(thumb["headers"].get_content_type() == "image/jpeg", "thumbnail must be JPEG")
|
||||
from image_probe import jpeg_size
|
||||
check(jpeg_size(thumb["bytes"]) == (256, 160), "actual JPEG size differs")
|
||||
|
||||
print(json.dumps({
|
||||
"attachment_id": att["id"],
|
||||
"message_id": msg["id"],
|
||||
"status_after_worker_restart": ready["status"],
|
||||
"thumbnail": {"width": v["width"], "height": v["height"],
|
||||
"size_bytes": v["size_bytes"], "sha256": v["sha256"]},
|
||||
"seconds_from_worker_start_to_ready": elapsed,
|
||||
}, ensure_ascii=False))
|
||||
print("PASS: queued job completed after API and worker restart.")
|
||||
finally:
|
||||
compose("up", "-d", "--no-deps", "worker")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run(main)
|
||||
Executable → Regular
@@ -0,0 +1,3 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
docker compose --env-file .env restart postgres s3 broker api1 api2 worker lb
|
||||
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
"C:\Program Files\Git\bin\bash.exe" "%~dp0restart.sh"
|
||||
Executable → Regular
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
: "${TARGET_INSTANCE_ID:?}"
|
||||
case "$TARGET_INSTANCE_ID" in
|
||||
api-1) SVC=api1 ;;
|
||||
api-2) SVC=api2 ;;
|
||||
*) echo "unknown instance alias: $TARGET_INSTANCE_ID" >&2; exit 1 ;;
|
||||
esac
|
||||
docker compose --env-file .env stop "$SVC"
|
||||
@@ -0,0 +1,2 @@
|
||||
@echo off
|
||||
"C:\Program Files\Git\bin\bash.exe" "%~dp0stop-one.sh"
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
import hashlib
|
||||
import secrets
|
||||
from datetime import datetime, timezone, timedelta
|
||||
from fastapi import Depends, Request
|
||||
from sqlalchemy.orm import Session
|
||||
from argon2 import PasswordHasher
|
||||
from argon2.exceptions import VerifyMismatchError
|
||||
from . import config
|
||||
from .db import get_db
|
||||
from .models import User, Session as DbSession
|
||||
from .errors import ApiError
|
||||
|
||||
_ph = PasswordHasher()
|
||||
_BEARER = 'Bearer realm="messenger"'
|
||||
|
||||
def hash_password(p: str) -> str:
|
||||
return _ph.hash(p)
|
||||
|
||||
def verify_password(h: str, p: str) -> bool:
|
||||
try:
|
||||
return _ph.verify(h, p)
|
||||
except VerifyMismatchError:
|
||||
return False
|
||||
|
||||
def new_session(db: Session, user: User) -> tuple[str, datetime]:
|
||||
token = secrets.token_urlsafe(48)
|
||||
token_hash = hashlib.sha256(token.encode()).hexdigest()
|
||||
expires = datetime.now(timezone.utc) + timedelta(seconds=config.SESSION_TTL_SECONDS)
|
||||
db.add(DbSession(token_hash=token_hash, user_id=user.id, expires_at=expires))
|
||||
db.commit()
|
||||
return token, expires
|
||||
|
||||
def current_user(request: Request, db: Session = Depends(get_db)) -> User:
|
||||
auth = request.headers.get("authorization", "")
|
||||
if not auth.lower().startswith("bearer "):
|
||||
raise ApiError(401, "unauthorized", "missing bearer token", challenge=_BEARER)
|
||||
token = auth[7:].strip()
|
||||
token_hash = hashlib.sha256(token.encode()).hexdigest()
|
||||
s = db.get(DbSession, token_hash)
|
||||
if not s:
|
||||
raise ApiError(401, "unauthorized", "invalid session", challenge=_BEARER)
|
||||
if s.expires_at <= datetime.now(timezone.utc):
|
||||
raise ApiError(401, "unauthorized", "expired session", challenge=_BEARER)
|
||||
u = db.get(User, s.user_id)
|
||||
if not u:
|
||||
raise ApiError(401, "unauthorized", "user gone", challenge=_BEARER)
|
||||
return u
|
||||
@@ -0,0 +1,36 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
def _read_secret(env_name: str, file_env: str, default=None):
|
||||
path = os.environ.get(file_env)
|
||||
if path:
|
||||
return Path(path).read_text().strip()
|
||||
return os.environ.get(env_name, default)
|
||||
|
||||
APP_HOST = os.environ.get("APP_HOST", "0.0.0.0")
|
||||
APP_PORT = int(os.environ.get("APP_PORT", "8080"))
|
||||
APP_ENV = os.environ.get("APP_ENV", "lab")
|
||||
SESSION_TTL_SECONDS = int(os.environ.get("SESSION_TTL_SECONDS", "3600"))
|
||||
|
||||
DATABASE_URL = _read_secret("DATABASE_URL", "DATABASE_URL_FILE")
|
||||
|
||||
S3_ENDPOINT = os.environ.get("S3_ENDPOINT", "http://s3:9000")
|
||||
S3_REGION = os.environ.get("S3_REGION", "us-east-1")
|
||||
S3_BUCKET = os.environ.get("S3_BUCKET", "messenger-attachments")
|
||||
S3_ACCESS_KEY = _read_secret("S3_ACCESS_KEY_ID", "S3_ACCESS_KEY_ID_FILE")
|
||||
S3_SECRET_KEY = _read_secret("S3_SECRET_ACCESS_KEY", "S3_SECRET_ACCESS_KEY_FILE")
|
||||
|
||||
MAX_UPLOAD_BYTES = int(os.environ.get("MAX_UPLOAD_BYTES", str(10 * 1024 * 1024)))
|
||||
MAX_IMAGE_PIXELS = int(os.environ.get("MAX_IMAGE_PIXELS", "40000000"))
|
||||
|
||||
INSTANCE_ID = os.environ.get("INSTANCE_ID", "api-unknown")
|
||||
|
||||
# --- lab5: durable broker + worker ---
|
||||
BROKER_URL = _read_secret("BROKER_URL", "BROKER_URL_FILE", "amqp://messenger:messenger@broker:5672/")
|
||||
QUEUE_NAME = os.environ.get("JOB_QUEUE", "attachment_jobs")
|
||||
STAGING_PREFIX = os.environ.get("STAGING_PREFIX", "staging")
|
||||
THUMBNAIL_SIZE = int(os.environ.get("THUMBNAIL_SIZE", "256"))
|
||||
THUMBNAIL_QUALITY = int(os.environ.get("THUMBNAIL_QUALITY", "85"))
|
||||
MAX_JOB_ATTEMPTS = int(os.environ.get("MAX_JOB_ATTEMPTS", "5"))
|
||||
OUTBOX_INTERVAL_SECONDS = float(os.environ.get("OUTBOX_INTERVAL_SECONDS", "2"))
|
||||
OUTBOX_GRACE_SECONDS = float(os.environ.get("OUTBOX_GRACE_SECONDS", "0"))
|
||||
@@ -0,0 +1,16 @@
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker, DeclarativeBase
|
||||
from . import config
|
||||
|
||||
engine = create_engine(config.DATABASE_URL, pool_pre_ping=True, pool_size=5, max_overflow=10)
|
||||
SessionLocal = sessionmaker(bind=engine, autoflush=False, expire_on_commit=False)
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
def get_db():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,29 @@
|
||||
from fastapi import Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
class ApiError(Exception):
|
||||
def __init__(self, status: int, code: str, message: str, challenge: str | None = None):
|
||||
self.status = status
|
||||
self.code = code
|
||||
self.message = message
|
||||
self.challenge = challenge
|
||||
|
||||
def _rid(request: Request) -> str:
|
||||
return getattr(request.state, "request_id", None) or "unknown"
|
||||
|
||||
async def api_error_handler(request: Request, exc: ApiError):
|
||||
headers = {"X-Request-Id": _rid(request)}
|
||||
if exc.challenge:
|
||||
headers["WWW-Authenticate"] = exc.challenge
|
||||
return JSONResponse(
|
||||
status_code=exc.status,
|
||||
content={"error": {"code": exc.code, "message": exc.message, "request_id": _rid(request)}},
|
||||
headers=headers,
|
||||
)
|
||||
|
||||
async def unhandled_handler(request: Request, exc: Exception):
|
||||
return JSONResponse(
|
||||
status_code=500,
|
||||
content={"error": {"code": "internal_error", "message": "internal error", "request_id": _rid(request)}},
|
||||
headers={"X-Request-Id": _rid(request)},
|
||||
)
|
||||
@@ -0,0 +1,26 @@
|
||||
import io
|
||||
from PIL import Image, UnidentifiedImageError
|
||||
from . import config
|
||||
from .errors import ApiError
|
||||
|
||||
def validate_image(data: bytes) -> tuple[str, int, int, int]:
|
||||
"""Возвращает (content_type, width, height, size)."""
|
||||
if not data:
|
||||
raise ApiError(400, "invalid_request", "empty body")
|
||||
if len(data) > config.MAX_UPLOAD_BYTES:
|
||||
raise ApiError(413, "payload_too_large", "file too large")
|
||||
head = data[:8]
|
||||
if head.startswith(b"\x89PNG\r\n\x1a\n"):
|
||||
ctype = "image/png"
|
||||
elif head.startswith(b"\xff\xd8\xff"):
|
||||
ctype = "image/jpeg"
|
||||
else:
|
||||
raise ApiError(415, "unsupported_media_type", "only PNG/JPEG")
|
||||
try:
|
||||
img = Image.open(io.BytesIO(data))
|
||||
w, h = img.size
|
||||
except UnidentifiedImageError:
|
||||
raise ApiError(415, "unsupported_media_type", "invalid image")
|
||||
if w * h > config.MAX_IMAGE_PIXELS:
|
||||
raise ApiError(413, "payload_too_large", "too many pixels")
|
||||
return ctype, w, h, len(data)
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
"""Broker access: durable job publication and the transactional-outbox pump.
|
||||
|
||||
The broker message carries only identifiers and metadata (never bytes or
|
||||
credentials). The job row in PostgreSQL is the source of truth; `published_at`
|
||||
turns the row into a small transactional outbox so a crash between COMMIT and
|
||||
publish is repaired by the background pump.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone, timedelta
|
||||
|
||||
import pika
|
||||
from sqlalchemy import select
|
||||
|
||||
from . import config
|
||||
from .db import SessionLocal
|
||||
from .models import AttachmentJob
|
||||
|
||||
log = logging.getLogger("jobs")
|
||||
|
||||
TYPE_V1 = "attachment.process.v1"
|
||||
|
||||
|
||||
def build_payload(job: AttachmentJob, attempt: int) -> dict:
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"job_id": str(job.id),
|
||||
"type": TYPE_V1,
|
||||
"attachment_id": str(job.attachment_id),
|
||||
"pipeline_version": job.pipeline_version,
|
||||
"attempt": attempt,
|
||||
"created_at": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
|
||||
|
||||
def publish(payload: dict) -> None:
|
||||
params = pika.URLParameters(config.BROKER_URL)
|
||||
params.heartbeat = 30
|
||||
params.blocked_connection_timeout = 30
|
||||
connection = pika.BlockingConnection(params)
|
||||
try:
|
||||
channel = connection.channel()
|
||||
channel.queue_declare(queue=config.QUEUE_NAME, durable=True)
|
||||
channel.confirm_delivery()
|
||||
channel.basic_publish(
|
||||
exchange="",
|
||||
routing_key=config.QUEUE_NAME,
|
||||
body=json.dumps(payload).encode("utf-8"),
|
||||
properties=pika.BasicProperties(
|
||||
delivery_mode=2, # persistent
|
||||
content_type="application/json",
|
||||
message_id=payload["job_id"],
|
||||
),
|
||||
mandatory=True,
|
||||
)
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
|
||||
def publish_job_row(job_id: uuid.UUID, attempt: int) -> None:
|
||||
with SessionLocal() as db:
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
if not job or job.state in ("done", "failed"):
|
||||
return
|
||||
payload = build_payload(job, attempt)
|
||||
publish(payload)
|
||||
with SessionLocal() as db:
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
if job:
|
||||
job.published_at = datetime.now(timezone.utc)
|
||||
db.commit()
|
||||
|
||||
|
||||
def pump_once(limit: int = 50) -> int:
|
||||
"""Publish queued rows whose outbox timestamp is still empty."""
|
||||
now = datetime.now(timezone.utc)
|
||||
cutoff = now - timedelta(seconds=config.OUTBOX_GRACE_SECONDS)
|
||||
published = 0
|
||||
with SessionLocal() as db:
|
||||
rows = (
|
||||
db.execute(
|
||||
select(AttachmentJob)
|
||||
.where(AttachmentJob.published_at.is_(None))
|
||||
.where(AttachmentJob.state == "queued")
|
||||
.where(AttachmentJob.created_at <= cutoff)
|
||||
.order_by(AttachmentJob.created_at)
|
||||
.limit(limit)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
pending = [(job.id, job.attempt + 1) for job in rows]
|
||||
for job_id, attempt in pending:
|
||||
try:
|
||||
publish_job_row(job_id, attempt)
|
||||
published += 1
|
||||
except Exception as exc: # broker may still be starting
|
||||
log.warning("outbox publish failed for %s: %s", job_id, exc)
|
||||
return published
|
||||
|
||||
|
||||
def start_outbox_thread() -> threading.Thread:
|
||||
def loop():
|
||||
while True:
|
||||
try:
|
||||
count = pump_once()
|
||||
if count:
|
||||
log.info("outbox published %d job(s)", count)
|
||||
except Exception as exc:
|
||||
log.warning("outbox pump error: %s", exc)
|
||||
time.sleep(config.OUTBOX_INTERVAL_SECONDS)
|
||||
|
||||
thread = threading.Thread(target=loop, name="outbox", daemon=True)
|
||||
thread.start()
|
||||
return thread
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
import uuid
|
||||
from contextlib import asynccontextmanager
|
||||
from fastapi import FastAPI, Request
|
||||
from . import config
|
||||
from .db import engine, Base
|
||||
from .errors import ApiError, api_error_handler, unhandled_handler
|
||||
from .s3 import ensure_bucket
|
||||
from . import jobs
|
||||
from .routers import health, users, sessions, chats, messages, attachments
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# create_all is not idempotent under parallel startup: two replicas
|
||||
# may race and one hits UniqueViolation on pg_type. Catch and re-run
|
||||
# once after a short delay; by then the other replica has finished.
|
||||
import asyncio
|
||||
from sqlalchemy.exc import IntegrityError, OperationalError
|
||||
|
||||
for attempt in range(5):
|
||||
try:
|
||||
Base.metadata.create_all(engine)
|
||||
break
|
||||
except (IntegrityError, OperationalError) as exc:
|
||||
print(f"WARN: create_all attempt {attempt+1} failed: {exc}")
|
||||
await asyncio.sleep(1.0)
|
||||
else:
|
||||
print("WARN: create_all failed after 5 attempts; continuing")
|
||||
|
||||
try:
|
||||
ensure_bucket()
|
||||
except Exception as exc:
|
||||
print(f"WARN: ensure_bucket failed: {exc}")
|
||||
jobs.start_outbox_thread()
|
||||
yield
|
||||
|
||||
app = FastAPI(lifespan=lifespan)
|
||||
app.add_exception_handler(ApiError, api_error_handler)
|
||||
app.add_exception_handler(Exception, unhandled_handler)
|
||||
|
||||
@app.middleware("http")
|
||||
async def add_request_id(request: Request, call_next):
|
||||
rid = request.headers.get("X-Request-Id") or uuid.uuid4().hex
|
||||
request.state.request_id = rid
|
||||
response = await call_next(request)
|
||||
response.headers["X-Request-Id"] = rid
|
||||
response.headers["X-Instance-Id"] = config.INSTANCE_ID
|
||||
return response
|
||||
|
||||
for r in (health, users, sessions, chats, messages, attachments):
|
||||
app.include_router(r.router)
|
||||
+142
@@ -0,0 +1,142 @@
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from sqlalchemy import String, DateTime, ForeignKey, Integer, UniqueConstraint, Index
|
||||
from sqlalchemy.dialects.postgresql import UUID as PGUUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from .db import Base
|
||||
|
||||
|
||||
def _uuid() -> uuid.UUID:
|
||||
return uuid.uuid4()
|
||||
|
||||
|
||||
def _now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class User(Base):
|
||||
__tablename__ = "users"
|
||||
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
|
||||
username: Mapped[str] = mapped_column(String(32), unique=True, index=True)
|
||||
display_name: Mapped[str] = mapped_column(String(100))
|
||||
password_hash: Mapped[str] = mapped_column(String(255))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
|
||||
|
||||
class Session(Base):
|
||||
__tablename__ = "sessions"
|
||||
token_hash: Mapped[str] = mapped_column(String(64), primary_key=True) # sha256(token) hex
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
expires_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), index=True)
|
||||
|
||||
|
||||
class Chat(Base):
|
||||
__tablename__ = "chats"
|
||||
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
|
||||
title: Mapped[str] = mapped_column(String(100))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True)
|
||||
|
||||
|
||||
class ChatMember(Base):
|
||||
__tablename__ = "chat_members"
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
|
||||
|
||||
class Message(Base):
|
||||
__tablename__ = "messages"
|
||||
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
sender_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id")
|
||||
)
|
||||
text: Mapped[str] = mapped_column(String(4000), default="")
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, index=True)
|
||||
|
||||
|
||||
class MessageAttachment(Base):
|
||||
__tablename__ = "message_attachments"
|
||||
message_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("messages.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
attachment_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("attachments.id"), primary_key=True
|
||||
)
|
||||
position: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
|
||||
class IdempotencyKey(Base):
|
||||
__tablename__ = "idempotency_keys"
|
||||
user_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
method: Mapped[str] = mapped_column(String(8), primary_key=True)
|
||||
path: Mapped[str] = mapped_column(String(255), primary_key=True)
|
||||
key: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
request_hash: Mapped[str] = mapped_column(String(64))
|
||||
response_json: Mapped[str] = mapped_column(String)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
|
||||
__table_args__ = (Index("ix_idem_created", "created_at"),)
|
||||
|
||||
|
||||
class Attachment(Base):
|
||||
__tablename__ = "attachments"
|
||||
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
|
||||
chat_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("chats.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
owner_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("users.id")
|
||||
)
|
||||
filename: Mapped[str] = mapped_column(String(255))
|
||||
content_type: Mapped[str] = mapped_column(String(32))
|
||||
size_bytes: Mapped[int] = mapped_column(Integer)
|
||||
sha256: Mapped[str] = mapped_column(String(64))
|
||||
s3_key: Mapped[str] = mapped_column(String(255))
|
||||
staging_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
status: Mapped[str] = mapped_column(String(16), default="ready")
|
||||
error_code: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class AttachmentJob(Base):
|
||||
"""Durable intent to process one attachment (transactional outbox row)."""
|
||||
__tablename__ = "attachment_jobs"
|
||||
id: Mapped[uuid.UUID] = mapped_column(PGUUID(as_uuid=True), primary_key=True, default=_uuid)
|
||||
attachment_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("attachments.id", ondelete="CASCADE"), unique=True
|
||||
)
|
||||
pipeline_version: Mapped[int] = mapped_column(Integer, default=1)
|
||||
state: Mapped[str] = mapped_column(String(16), default="queued") # queued|processing|done|failed
|
||||
attempt: Mapped[int] = mapped_column(Integer, default=0)
|
||||
published_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
last_error: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now, onupdate=_now)
|
||||
|
||||
|
||||
class AttachmentVariant(Base):
|
||||
__tablename__ = "attachment_variants"
|
||||
attachment_id: Mapped[uuid.UUID] = mapped_column(
|
||||
PGUUID(as_uuid=True), ForeignKey("attachments.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
name: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
content_type: Mapped[str] = mapped_column(String(32))
|
||||
size_bytes: Mapped[int] = mapped_column(Integer)
|
||||
width: Mapped[int] = mapped_column(Integer)
|
||||
height: Mapped[int] = mapped_column(Integer)
|
||||
sha256: Mapped[str] = mapped_column(String(64))
|
||||
s3_key: Mapped[str] = mapped_column(String(255))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=_now)
|
||||
@@ -0,0 +1,119 @@
|
||||
import hashlib
|
||||
import uuid
|
||||
from fastapi import APIRouter, Depends, Request, Query
|
||||
from fastapi.responses import Response
|
||||
from sqlalchemy.orm import Session
|
||||
from ..db import get_db
|
||||
from ..models import Attachment, ChatMember, AttachmentJob, AttachmentVariant
|
||||
from ..auth import current_user
|
||||
from ..errors import ApiError
|
||||
from .. import s3, config, jobs
|
||||
from ..images import validate_image
|
||||
from ..schemas import AttachmentOut, VariantOut
|
||||
|
||||
router = APIRouter(prefix="/api/v1", tags=["attachments"])
|
||||
|
||||
def _bad_filename(name: str) -> bool:
|
||||
return (not name) or len(name) > 255 or any(c in name for c in ("/", "\\", "\x00")) or any(ord(c) < 32 for c in name)
|
||||
|
||||
def _member_chat_or_404(db, chat_id, user_id):
|
||||
if not db.get(ChatMember, (chat_id, user_id)):
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
|
||||
def _attachment_for_member(db, att_id, user_id) -> Attachment:
|
||||
att = db.get(Attachment, att_id)
|
||||
if not att:
|
||||
raise ApiError(404, "not_found", "attachment not found")
|
||||
_member_chat_or_404(db, att.chat_id, user_id)
|
||||
return att
|
||||
|
||||
def _attachment_out(db, att: Attachment) -> AttachmentOut:
|
||||
variants = db.query(AttachmentVariant).filter(
|
||||
AttachmentVariant.attachment_id == att.id
|
||||
).all()
|
||||
return AttachmentOut(
|
||||
id=att.id, chat_id=att.chat_id, owner_id=att.owner_id,
|
||||
filename=att.filename, content_type=att.content_type,
|
||||
size_bytes=att.size_bytes, sha256=att.sha256,
|
||||
status=att.status, error_code=att.error_code,
|
||||
variants=[VariantOut.model_validate(v) for v in variants],
|
||||
created_at=att.created_at, updated_at=att.updated_at,
|
||||
)
|
||||
|
||||
@router.post("/chats/{chat_id}/attachments", status_code=202, response_model=AttachmentOut)
|
||||
async def upload(chat_id: uuid.UUID,
|
||||
filename: str = Query(..., min_length=1, max_length=255),
|
||||
request: Request = None,
|
||||
db: Session = Depends(get_db),
|
||||
user=Depends(current_user)):
|
||||
_member_chat_or_404(db, chat_id, user.id)
|
||||
if _bad_filename(filename):
|
||||
raise ApiError(400, "invalid_request", "invalid filename")
|
||||
ctype_header = request.headers.get("content-type", "").split(";")[0].strip()
|
||||
if ctype_header not in ("image/png", "image/jpeg"):
|
||||
raise ApiError(415, "unsupported_media_type", "expected image/png or image/jpeg")
|
||||
data = await request.body()
|
||||
ctype, w, h, size = validate_image(data)
|
||||
if ctype != ctype_header:
|
||||
raise ApiError(415, "unsupported_media_type", "content-type mismatch with file")
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
|
||||
att_id = uuid.uuid4()
|
||||
final_key = f"chats/{chat_id}/attachments/{att_id}"
|
||||
staging_key = f"{config.STAGING_PREFIX}/{att_id}"
|
||||
|
||||
# 1. Durable staging: the original must survive an API restart before 202.
|
||||
s3.client().put_object(Bucket=config.S3_BUCKET, Key=staging_key, Body=data, ContentType=ctype)
|
||||
|
||||
# 2. Metadata + durable job intent in one transaction (transactional outbox).
|
||||
att = Attachment(id=att_id, chat_id=chat_id, owner_id=user.id, filename=filename,
|
||||
content_type=ctype, size_bytes=size, sha256=digest, s3_key=final_key,
|
||||
staging_key=staging_key, status="queued")
|
||||
job = AttachmentJob(attachment_id=att_id, pipeline_version=1, state="queued", attempt=0)
|
||||
try:
|
||||
db.add(att)
|
||||
db.add(job)
|
||||
db.commit()
|
||||
db.refresh(att)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
# Compensating delete for the staging object; the DB has no intent.
|
||||
try:
|
||||
s3.client().delete_object(Bucket=config.S3_BUCKET, Key=staging_key)
|
||||
except Exception:
|
||||
pass
|
||||
raise
|
||||
|
||||
# 3. Handoff to the broker. If this fails, the row stays unpublished and
|
||||
# the outbox pump retries it; the client still got a durable 202.
|
||||
from starlette.concurrency import run_in_threadpool
|
||||
try:
|
||||
await run_in_threadpool(jobs.publish_job_row, job.id, 1)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return _attachment_out(db, att)
|
||||
|
||||
@router.get("/attachments/{att_id}", response_model=AttachmentOut)
|
||||
def meta(att_id: uuid.UUID, db=Depends(get_db), user=Depends(current_user)):
|
||||
return _attachment_out(db, _attachment_for_member(db, att_id, user.id))
|
||||
|
||||
@router.get("/attachments/{att_id}/content")
|
||||
def content(att_id: uuid.UUID, variant: str | None = None,
|
||||
db=Depends(get_db), user=Depends(current_user)):
|
||||
att = _attachment_for_member(db, att_id, user.id)
|
||||
if att.status == "ready":
|
||||
if variant in (None, "original"):
|
||||
key, media_type = att.s3_key, att.content_type
|
||||
else:
|
||||
row = db.get(AttachmentVariant, (att.id, variant))
|
||||
if not row:
|
||||
raise ApiError(404, "not_found", "variant not found")
|
||||
key, media_type = row.s3_key, row.content_type
|
||||
elif att.status in ("queued", "processing"):
|
||||
raise ApiError(409, "conflict", "attachment is not ready")
|
||||
else:
|
||||
raise ApiError(409, "conflict", "attachment failed")
|
||||
obj = s3.client().get_object(Bucket=config.S3_BUCKET, Key=key)
|
||||
body = obj["Body"].read()
|
||||
return Response(content=body, media_type=media_type)
|
||||
@@ -0,0 +1,44 @@
|
||||
import uuid
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from ..db import get_db
|
||||
from ..models import User, Chat, ChatMember
|
||||
from ..schemas import CreateChatIn, ChatOut, ChatPage
|
||||
from ..errors import ApiError
|
||||
from ..auth import current_user
|
||||
|
||||
router = APIRouter(prefix="/api/v1/chats", tags=["chats"])
|
||||
|
||||
def _chat_out(db, chat) -> ChatOut:
|
||||
members = [m.user_id for m in db.query(ChatMember).filter(ChatMember.chat_id == chat.id).all()]
|
||||
return ChatOut(id=chat.id, title=chat.title, member_ids=members, created_at=chat.created_at)
|
||||
|
||||
@router.post("", status_code=201, response_model=ChatOut)
|
||||
def create_chat(body: CreateChatIn, db: Session = Depends(get_db), user: User = Depends(current_user)):
|
||||
if user.id not in body.member_ids:
|
||||
raise ApiError(400, "invalid_request", "caller must be a member")
|
||||
found = db.query(User.id).filter(User.id.in_(body.member_ids)).all()
|
||||
if len(found) != len(set(body.member_ids)):
|
||||
raise ApiError(400, "invalid_request", "unknown member")
|
||||
chat = Chat(title=body.title)
|
||||
db.add(chat); db.flush()
|
||||
for uid in body.member_ids:
|
||||
db.add(ChatMember(chat_id=chat.id, user_id=uid))
|
||||
db.commit(); db.refresh(chat)
|
||||
return _chat_out(db, chat)
|
||||
|
||||
@router.get("", response_model=ChatPage)
|
||||
def list_chats(db: Session = Depends(get_db), user: User = Depends(current_user)):
|
||||
rows = db.query(Chat).join(ChatMember, ChatMember.chat_id == Chat.id)\
|
||||
.filter(ChatMember.user_id == user.id)\
|
||||
.order_by(Chat.created_at, Chat.id).limit(100).all()
|
||||
return ChatPage(items=[_chat_out(db, c) for c in rows], next_cursor=None)
|
||||
|
||||
@router.get("/{chat_id}", response_model=ChatOut)
|
||||
def get_chat(chat_id: uuid.UUID, db: Session = Depends(get_db), user: User = Depends(current_user)):
|
||||
if not db.get(ChatMember, (chat_id, user.id)):
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
chat = db.get(Chat, chat_id)
|
||||
if not chat:
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
return _chat_out(db, chat)
|
||||
@@ -0,0 +1,19 @@
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
from ..db import get_db
|
||||
from ..errors import ApiError
|
||||
|
||||
router = APIRouter(prefix="/health", tags=["health"])
|
||||
|
||||
@router.get("/live")
|
||||
def live():
|
||||
return {"status": "ok"}
|
||||
|
||||
@router.get("/ready")
|
||||
def ready(db: Session = Depends(get_db)):
|
||||
try:
|
||||
db.execute(text("SELECT 1"))
|
||||
except Exception:
|
||||
raise ApiError(503, "unavailable", "database unavailable")
|
||||
return {"status": "ok"}
|
||||
@@ -0,0 +1,147 @@
|
||||
import uuid
|
||||
import json
|
||||
import hashlib
|
||||
import base64
|
||||
from datetime import datetime
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from ..db import get_db
|
||||
from ..models import (
|
||||
User,
|
||||
ChatMember,
|
||||
Message,
|
||||
MessageAttachment,
|
||||
IdempotencyKey,
|
||||
Attachment,
|
||||
)
|
||||
from ..schemas import CreateMessageIn, MessageOut, MessagePage
|
||||
from ..errors import ApiError
|
||||
from ..auth import current_user
|
||||
|
||||
router = APIRouter(prefix="/api/v1/chats/{chat_id}/messages", tags=["messages"])
|
||||
|
||||
|
||||
def _msg_out(db: Session, m: Message) -> MessageOut:
|
||||
atts = [
|
||||
a.attachment_id
|
||||
for a in db.query(MessageAttachment)
|
||||
.filter(MessageAttachment.message_id == m.id)
|
||||
.order_by(MessageAttachment.position)
|
||||
.all()
|
||||
]
|
||||
return MessageOut(
|
||||
id=m.id,
|
||||
chat_id=m.chat_id,
|
||||
sender_id=m.sender_id,
|
||||
text=m.text,
|
||||
created_at=m.created_at,
|
||||
attachment_ids=atts,
|
||||
)
|
||||
|
||||
|
||||
def _require_member(db: Session, chat_id: uuid.UUID, user_id: uuid.UUID) -> None:
|
||||
if not db.get(ChatMember, (chat_id, user_id)):
|
||||
raise ApiError(404, "not_found", "chat not found")
|
||||
|
||||
|
||||
@router.post("", status_code=201, response_model=MessageOut)
|
||||
def send(
|
||||
chat_id: uuid.UUID,
|
||||
body: CreateMessageIn,
|
||||
request: Request,
|
||||
db: Session = Depends(get_db),
|
||||
user: User = Depends(current_user),
|
||||
):
|
||||
_require_member(db, chat_id, user.id)
|
||||
|
||||
key = request.headers.get("idempotency-key", "")
|
||||
if not key or len(key) > 128:
|
||||
raise ApiError(400, "invalid_request", "Idempotency-Key required")
|
||||
|
||||
if not body.text.strip() and not body.attachment_ids:
|
||||
raise ApiError(400, "invalid_request", "empty message")
|
||||
|
||||
for aid in body.attachment_ids:
|
||||
a = db.get(Attachment, aid)
|
||||
if not a or a.chat_id != chat_id:
|
||||
raise ApiError(404, "not_found", "attachment not found")
|
||||
|
||||
path = f"/api/v1/chats/{chat_id}/messages"
|
||||
body_hash = hashlib.sha256(
|
||||
json.dumps(body.model_dump(), sort_keys=True, default=str).encode()
|
||||
).hexdigest()
|
||||
|
||||
existing = db.get(IdempotencyKey, (user.id, "POST", path, key))
|
||||
if existing:
|
||||
if existing.request_hash != body_hash:
|
||||
raise ApiError(409, "conflict", "idempotency key reused with different body")
|
||||
return json.loads(existing.response_json)
|
||||
|
||||
m = Message(chat_id=chat_id, sender_id=user.id, text=body.text)
|
||||
db.add(m)
|
||||
db.flush() # чтобы появился m.id
|
||||
|
||||
for i, aid in enumerate(body.attachment_ids):
|
||||
db.add(MessageAttachment(message_id=m.id, attachment_id=aid, position=i))
|
||||
db.flush() # чтобы _msg_out увидел MessageAttachment
|
||||
|
||||
out = _msg_out(db, m)
|
||||
|
||||
db.add(
|
||||
IdempotencyKey(
|
||||
user_id=user.id,
|
||||
method="POST",
|
||||
path=path,
|
||||
key=key,
|
||||
request_hash=body_hash,
|
||||
response_json=json.dumps(out.model_dump(), default=str),
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
return out
|
||||
|
||||
|
||||
@router.get("", response_model=MessagePage)
|
||||
def list_messages(
|
||||
chat_id: uuid.UUID,
|
||||
limit: int = 50,
|
||||
cursor: str | None = None,
|
||||
db: Session = Depends(get_db),
|
||||
user: User = Depends(current_user),
|
||||
):
|
||||
_require_member(db, chat_id, user.id)
|
||||
|
||||
if limit < 1 or limit > 100:
|
||||
raise ApiError(400, "invalid_request", "bad limit")
|
||||
|
||||
q = (
|
||||
db.query(Message)
|
||||
.filter(Message.chat_id == chat_id)
|
||||
.order_by(Message.created_at, Message.id)
|
||||
)
|
||||
|
||||
if cursor:
|
||||
try:
|
||||
c = json.loads(base64.urlsafe_b64decode(cursor.encode()).decode())
|
||||
ts = datetime.fromisoformat(c["t"])
|
||||
mid = uuid.UUID(c["i"])
|
||||
q = q.filter(
|
||||
(Message.created_at > ts)
|
||||
| ((Message.created_at == ts) & (Message.id > mid))
|
||||
)
|
||||
except Exception:
|
||||
raise ApiError(400, "invalid_request", "bad cursor")
|
||||
|
||||
rows = q.limit(limit + 1).all()
|
||||
has_more = len(rows) > limit
|
||||
rows = rows[:limit]
|
||||
|
||||
next_cursor = None
|
||||
if has_more:
|
||||
last = rows[-1]
|
||||
payload = json.dumps({"t": last.created_at.isoformat(), "i": str(last.id)}).encode()
|
||||
next_cursor = base64.urlsafe_b64encode(payload).decode()
|
||||
|
||||
return MessagePage(items=[_msg_out(db, m) for m in rows], next_cursor=next_cursor)
|
||||
@@ -0,0 +1,41 @@
|
||||
import base64
|
||||
import hashlib
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from sqlalchemy.orm import Session
|
||||
from ..db import get_db
|
||||
from ..models import User, Session as DbSession
|
||||
from ..schemas import SessionOut
|
||||
from ..errors import ApiError
|
||||
from ..auth import verify_password, new_session, current_user
|
||||
|
||||
router = APIRouter(prefix="/api/v1/auth/sessions", tags=["sessions"])
|
||||
|
||||
_BASIC = 'Basic realm="messenger", charset="UTF-8"'
|
||||
_BEARER = 'Bearer realm="messenger"'
|
||||
|
||||
@router.post("", status_code=201, response_model=SessionOut)
|
||||
def login(request: Request, db: Session = Depends(get_db)):
|
||||
auth = request.headers.get("authorization", "")
|
||||
if not auth.lower().startswith("basic "):
|
||||
raise ApiError(401, "unauthorized", "basic required", challenge=_BASIC)
|
||||
try:
|
||||
raw = base64.b64decode(auth[6:].strip()).decode("utf-8")
|
||||
username, password = raw.split(":", 1)
|
||||
except Exception:
|
||||
raise ApiError(401, "unauthorized", "bad credentials", challenge=_BASIC)
|
||||
user = db.query(User).filter(User.username == username).first()
|
||||
if not user or not verify_password(user.password_hash, password):
|
||||
raise ApiError(401, "unauthorized", "bad credentials", challenge=_BASIC)
|
||||
token, expires = new_session(db, user)
|
||||
return {"token": token, "token_type": "Bearer", "expires_at": expires, "user": user}
|
||||
|
||||
@router.delete("/current", status_code=204)
|
||||
def logout(request: Request, db: Session = Depends(get_db), user: User = Depends(current_user)):
|
||||
auth = request.headers.get("authorization", "")
|
||||
token = auth[7:].strip()
|
||||
h = hashlib.sha256(token.encode()).hexdigest()
|
||||
s = db.get(DbSession, h)
|
||||
if s:
|
||||
db.delete(s)
|
||||
db.commit()
|
||||
return None
|
||||
@@ -0,0 +1,27 @@
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy.orm import Session
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from ..db import get_db
|
||||
from ..models import User
|
||||
from ..schemas import CreateUserIn, UserOut
|
||||
from ..errors import ApiError
|
||||
from ..auth import hash_password, current_user
|
||||
|
||||
router = APIRouter(prefix="/api/v1/users", tags=["users"])
|
||||
|
||||
@router.post("", status_code=201, response_model=UserOut)
|
||||
def register(body: CreateUserIn, db: Session = Depends(get_db)):
|
||||
u = User(username=body.username, display_name=body.display_name,
|
||||
password_hash=hash_password(body.password))
|
||||
db.add(u)
|
||||
try:
|
||||
db.commit()
|
||||
except IntegrityError:
|
||||
db.rollback()
|
||||
raise ApiError(409, "conflict", "username taken")
|
||||
db.refresh(u)
|
||||
return u
|
||||
|
||||
@router.get("/me", response_model=UserOut)
|
||||
def me(user: User = Depends(current_user)):
|
||||
return user
|
||||
@@ -0,0 +1,36 @@
|
||||
import boto3
|
||||
from botocore.client import Config
|
||||
from botocore.exceptions import ClientError
|
||||
from . import config
|
||||
|
||||
_client = None
|
||||
|
||||
def client():
|
||||
global _client
|
||||
if _client is None:
|
||||
_client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=config.S3_ENDPOINT,
|
||||
region_name=config.S3_REGION,
|
||||
aws_access_key_id=config.S3_ACCESS_KEY,
|
||||
aws_secret_access_key=config.S3_SECRET_KEY,
|
||||
config=Config(
|
||||
signature_version="s3v4",
|
||||
s3={"addressing_style": "path"},
|
||||
retries={"max_attempts": 2, "mode": "standard"},
|
||||
connect_timeout=3,
|
||||
read_timeout=10,
|
||||
),
|
||||
)
|
||||
return _client
|
||||
|
||||
def ensure_bucket():
|
||||
c = client()
|
||||
try:
|
||||
c.head_bucket(Bucket=config.S3_BUCKET)
|
||||
except ClientError as e:
|
||||
code = e.response.get("Error", {}).get("Code", "")
|
||||
if code in ("404", "NoSuchBucket", "NotFound"):
|
||||
c.create_bucket(Bucket=config.S3_BUCKET)
|
||||
else:
|
||||
raise
|
||||
@@ -0,0 +1,76 @@
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from pydantic import BaseModel, Field, ConfigDict
|
||||
|
||||
class UserOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
id: uuid.UUID
|
||||
username: str
|
||||
display_name: str
|
||||
created_at: datetime
|
||||
|
||||
class CreateUserIn(BaseModel):
|
||||
username: str = Field(pattern=r"^[a-z0-9_]{3,32}$")
|
||||
display_name: str = Field(min_length=1, max_length=100)
|
||||
password: str = Field(min_length=12, max_length=128)
|
||||
|
||||
class SessionOut(BaseModel):
|
||||
token: str
|
||||
token_type: str = "Bearer"
|
||||
expires_at: datetime
|
||||
user: UserOut
|
||||
|
||||
class ChatOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
id: uuid.UUID
|
||||
title: str
|
||||
member_ids: list[uuid.UUID]
|
||||
created_at: datetime
|
||||
|
||||
class CreateChatIn(BaseModel):
|
||||
title: str = Field(min_length=1, max_length=100)
|
||||
member_ids: list[uuid.UUID] = Field(min_length=1, max_length=100)
|
||||
|
||||
class MessageOut(BaseModel):
|
||||
id: uuid.UUID
|
||||
chat_id: uuid.UUID
|
||||
sender_id: uuid.UUID
|
||||
text: str
|
||||
created_at: datetime
|
||||
attachment_ids: list[uuid.UUID]
|
||||
|
||||
class CreateMessageIn(BaseModel):
|
||||
text: str = Field(default="", max_length=4000)
|
||||
attachment_ids: list[uuid.UUID] = Field(default_factory=list, max_length=10)
|
||||
|
||||
class MessagePage(BaseModel):
|
||||
items: list[MessageOut]
|
||||
next_cursor: Optional[str] = None
|
||||
|
||||
class ChatPage(BaseModel):
|
||||
items: list[ChatOut]
|
||||
next_cursor: Optional[str] = None
|
||||
|
||||
class VariantOut(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
name: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
width: int
|
||||
height: int
|
||||
sha256: str
|
||||
|
||||
class AttachmentOut(BaseModel):
|
||||
id: uuid.UUID
|
||||
chat_id: uuid.UUID
|
||||
owner_id: uuid.UUID
|
||||
filename: str
|
||||
content_type: str
|
||||
size_bytes: int
|
||||
sha256: str
|
||||
status: str = "ready"
|
||||
variants: list[VariantOut] = []
|
||||
error_code: Optional[str] = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Deterministic image variants (lab5: one thumbnail per the contract)."""
|
||||
import hashlib
|
||||
import io
|
||||
|
||||
from PIL import Image, ImageOps
|
||||
|
||||
from . import config
|
||||
|
||||
|
||||
def _flatten_on_white(img: Image.Image) -> Image.Image:
|
||||
if img.mode in ("RGBA", "LA") or (img.mode == "P" and "transparency" in img.info):
|
||||
rgba = img.convert("RGBA")
|
||||
background = Image.new("RGBA", rgba.size, (255, 255, 255, 255))
|
||||
return Image.alpha_composite(background, rgba).convert("RGB")
|
||||
return img.convert("RGB")
|
||||
|
||||
|
||||
def make_thumbnail(data: bytes) -> tuple[bytes, int, int, str]:
|
||||
"""Fit the image into 256x256 (no upscaling), JPEG on white.
|
||||
|
||||
Returns (jpeg_bytes, width, height, sha256_hex).
|
||||
"""
|
||||
img = Image.open(io.BytesIO(data))
|
||||
img = ImageOps.exif_transpose(img)
|
||||
img = _flatten_on_white(img)
|
||||
|
||||
limit = config.THUMBNAIL_SIZE
|
||||
width, height = img.size
|
||||
scale = min(limit / width, limit / height, 1.0)
|
||||
new_width = max(1, int(width * scale))
|
||||
new_height = max(1, int(height * scale))
|
||||
if (new_width, new_height) != (width, height):
|
||||
img = img.resize((new_width, new_height), Image.LANCZOS)
|
||||
|
||||
out = io.BytesIO()
|
||||
img.save(out, "JPEG", quality=config.THUMBNAIL_QUALITY, optimize=True)
|
||||
payload = out.getvalue()
|
||||
return payload, new_width, new_height, hashlib.sha256(payload).hexdigest()
|
||||
+197
@@ -0,0 +1,197 @@
|
||||
"""Durable attachment worker (lab5).
|
||||
|
||||
Consumes ``attachment.process.v1`` messages from a persistent queue, performs
|
||||
the heavy image work outside the HTTP path, and records the result atomically
|
||||
in PostgreSQL. Delivery is at-least-once; the effect is idempotent by
|
||||
``(attachment_id, pipeline_version, variant)`` and every state transition is
|
||||
committed before the broker ack.
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import signal
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import pika
|
||||
from sqlalchemy import select
|
||||
|
||||
from . import config, s3
|
||||
from .db import SessionLocal
|
||||
from .errors import ApiError
|
||||
from .images import validate_image
|
||||
from .models import Attachment, AttachmentJob, AttachmentVariant
|
||||
from .thumbnails import make_thumbnail
|
||||
|
||||
log = logging.getLogger("worker")
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
|
||||
|
||||
_stop = False
|
||||
|
||||
|
||||
def _handle_signal(signum, frame):
|
||||
global _stop
|
||||
_stop = True
|
||||
log.info("stop requested")
|
||||
|
||||
|
||||
def _failure_code(exc: Exception) -> tuple[str, bool]:
|
||||
"""Return (safe error_code, permanent)."""
|
||||
if isinstance(exc, ApiError):
|
||||
if exc.status in (400, 413, 415):
|
||||
return "invalid_image", True
|
||||
return "processing_failed", False
|
||||
if isinstance(exc, (OSError, ValueError)):
|
||||
return "invalid_image", True
|
||||
return "processing_failed", False
|
||||
|
||||
|
||||
def _set_state(job_id: uuid.UUID, attachment_id: uuid.UUID, job_state: str,
|
||||
att_state: str | None = None, attempt: int | None = None,
|
||||
error: str | None = None) -> None:
|
||||
with SessionLocal() as db:
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
if not job:
|
||||
return
|
||||
job.state = job_state
|
||||
if attempt is not None:
|
||||
job.attempt = max(job.attempt or 0, attempt)
|
||||
if error is not None:
|
||||
job.last_error = error[:255]
|
||||
att = db.get(Attachment, attachment_id)
|
||||
if att and att_state:
|
||||
att.status = att_state
|
||||
att.error_code = error if att_state == "failed" else None
|
||||
db.commit()
|
||||
|
||||
|
||||
def process(job_id: uuid.UUID, attachment_id: uuid.UUID, attempt: int) -> None:
|
||||
with SessionLocal() as db:
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
att = db.get(Attachment, attachment_id)
|
||||
if not job or not att:
|
||||
log.warning("job %s or attachment %s is gone; drop", job_id, attachment_id)
|
||||
return
|
||||
if att.status == "ready":
|
||||
job.state = "done"
|
||||
db.commit()
|
||||
log.info("attachment %s already ready; duplicate delivery is a no-op", attachment_id)
|
||||
return
|
||||
if att.status == "failed":
|
||||
job.state = "failed"
|
||||
db.commit()
|
||||
log.info("attachment %s already failed; drop duplicate", attachment_id)
|
||||
return
|
||||
job.state = "processing"
|
||||
job.attempt = max(job.attempt or 0, attempt)
|
||||
att.status = "processing"
|
||||
db.commit()
|
||||
pipeline_version = job.pipeline_version
|
||||
staging_key = att.staging_key or f"{config.STAGING_PREFIX}/{attachment_id}"
|
||||
final_key = att.s3_key
|
||||
|
||||
try:
|
||||
with SessionLocal() as db:
|
||||
att = db.get(Attachment, attachment_id)
|
||||
declared_type = att.content_type if att else None
|
||||
data = s3.client().get_object(Bucket=config.S3_BUCKET, Key=staging_key)["Body"].read()
|
||||
ctype, _w, _h, _size = validate_image(data)
|
||||
s3.client().put_object(Bucket=config.S3_BUCKET, Key=final_key, Body=data,
|
||||
ContentType=declared_type or ctype)
|
||||
|
||||
thumb, tw, th, tsha = make_thumbnail(data)
|
||||
thumb_key = f"variants/{attachment_id}/thumbnail"
|
||||
s3.client().put_object(Bucket=config.S3_BUCKET, Key=thumb_key, Body=thumb, ContentType="image/jpeg")
|
||||
|
||||
with SessionLocal() as db:
|
||||
att = db.get(Attachment, attachment_id)
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
existing = db.get(AttachmentVariant, (attachment_id, "thumbnail"))
|
||||
if existing is None:
|
||||
db.add(AttachmentVariant(
|
||||
attachment_id=attachment_id, name="thumbnail",
|
||||
content_type="image/jpeg", size_bytes=len(thumb),
|
||||
width=tw, height=th, sha256=tsha, s3_key=thumb_key,
|
||||
))
|
||||
elif existing.s3_key != thumb_key:
|
||||
# A newer pipeline revision owns the object; never overwrite it.
|
||||
log.warning("variant for %s written by another pipeline; keeping existing", attachment_id)
|
||||
att.status = "ready"
|
||||
att.error_code = None
|
||||
job.state = "done"
|
||||
job.last_error = None
|
||||
job.attempt = max(job.attempt or 0, attempt)
|
||||
db.commit()
|
||||
log.info("attachment %s ready (pipeline v%s, attempt %s)", attachment_id, pipeline_version, attempt)
|
||||
try:
|
||||
s3.client().delete_object(Bucket=config.S3_BUCKET, Key=staging_key)
|
||||
except Exception:
|
||||
pass
|
||||
except Exception as exc:
|
||||
code, permanent = _failure_code(exc)
|
||||
log.exception("job %s attempt %s failed: %s", job_id, attempt, exc)
|
||||
if permanent or attempt >= config.MAX_JOB_ATTEMPTS:
|
||||
_set_state(job_id, attachment_id, "failed", att_state="failed",
|
||||
attempt=attempt, error=code)
|
||||
return
|
||||
_set_state(job_id, attachment_id, "queued", att_state="queued",
|
||||
attempt=attempt, error=code)
|
||||
raise
|
||||
|
||||
|
||||
def handle(channel, method, properties, body):
|
||||
try:
|
||||
message = json.loads(body)
|
||||
job_id = uuid.UUID(message["job_id"])
|
||||
attachment_id = uuid.UUID(message["attachment_id"])
|
||||
attempt = int(message.get("attempt", 1))
|
||||
except Exception as exc:
|
||||
log.warning("poison message dropped: %s", exc)
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
return
|
||||
try:
|
||||
process(job_id, attachment_id, attempt)
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
except Exception:
|
||||
# Retry by re-publishing with an incremented attempt; ack only after
|
||||
# the job row has been moved back to `queued`.
|
||||
from .jobs import build_payload, publish
|
||||
with SessionLocal() as db:
|
||||
job = db.get(AttachmentJob, job_id)
|
||||
if job and job.state == "queued":
|
||||
payload = build_payload(job, job.attempt + 1)
|
||||
else:
|
||||
payload = None
|
||||
if payload:
|
||||
try:
|
||||
publish(payload)
|
||||
except Exception as publish_exc:
|
||||
log.error("retry publish failed, message will be requeued: %s", publish_exc)
|
||||
channel.basic_nack(delivery_tag=method.delivery_tag, requeue=True)
|
||||
return
|
||||
channel.basic_ack(delivery_tag=method.delivery_tag)
|
||||
|
||||
|
||||
def main():
|
||||
signal.signal(signal.SIGTERM, _handle_signal)
|
||||
signal.signal(signal.SIGINT, _handle_signal)
|
||||
params = pika.URLParameters(config.BROKER_URL)
|
||||
params.heartbeat = 30
|
||||
while not _stop:
|
||||
try:
|
||||
connection = pika.BlockingConnection(params)
|
||||
channel = connection.channel()
|
||||
channel.queue_declare(queue=config.QUEUE_NAME, durable=True)
|
||||
channel.basic_qos(prefetch_count=1)
|
||||
channel.basic_consume(queue=config.QUEUE_NAME, on_message_callback=handle, auto_ack=False)
|
||||
log.info("worker consuming from %s", config.QUEUE_NAME)
|
||||
while not _stop:
|
||||
connection.process_data_events(time_limit=1)
|
||||
connection.close()
|
||||
except Exception as exc:
|
||||
log.warning("broker connection problem, retrying: %s", exc)
|
||||
time.sleep(3)
|
||||
log.info("worker stopped")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,129 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Explicit operator-authored hook; tokens remain in memory, volumes are never removed by this test."""
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
import uuid
|
||||
from urllib.parse import quote
|
||||
from client import Client, Failure, LAB, arguments, check, run
|
||||
|
||||
|
||||
def history(c, actor, chat):
|
||||
path = '/api/v1/chats/'+chat['id']+'/messages?limit=100'
|
||||
result = []
|
||||
seen = set()
|
||||
for _ in range(1000):
|
||||
page = c.call('GET',path,actor=actor,schema='MessagePage')['json']
|
||||
result.extend(page['items'])
|
||||
cursor = page['next_cursor']
|
||||
if cursor is None:
|
||||
return result
|
||||
check(cursor not in seen,'cursor cycle')
|
||||
seen.add(cursor)
|
||||
path = '/api/v1/chats/'+chat['id']+'/messages?limit=100&cursor='+quote(cursor,safe='')
|
||||
raise Failure('history pagination exceeded safety bound')
|
||||
|
||||
|
||||
def main():
|
||||
def configure(p):
|
||||
p.add_argument('--mode', choices=['persistence','failover'], required=True)
|
||||
p.add_argument('--action-script',required=True,help='Executable script authored by student; restarts stack or stops ONE API replica')
|
||||
args = arguments('Explicit restart/failover scenario; modifies only synthetic API data',configure)
|
||||
check(LAB>=2,'persistence requires lab2+')
|
||||
check(args.mode!='failover' or LAB>=3,'failover requires lab3+')
|
||||
script = Path(args.action_script).expanduser().resolve()
|
||||
check(script.is_file() and os.access(script,os.X_OK),'action script must exist and be executable (chmod +x)')
|
||||
check('.example.' not in script.name,'copy and implement the example hook first')
|
||||
c = Client(args.base_url,args.ca_file,timeout=10)
|
||||
alice,bob,eve,chat = c.bootstrap()
|
||||
sent = c.message(alice,chat,'Сохранить при отказе')
|
||||
original = sent['json']
|
||||
target = sent['headers'].get('X-Instance-Id','')
|
||||
attachment = content = None
|
||||
if LAB>=4:
|
||||
attachment,content = c.attachment(alice,chat)
|
||||
if LAB>=5:
|
||||
attachment = c.wait_ready(alice,attachment)
|
||||
env = dict(os.environ)
|
||||
env['TARGET_INSTANCE_ID'] = target
|
||||
observed = set()
|
||||
if args.mode=='failover':
|
||||
for _ in range(40):
|
||||
observed.add(c.call('GET','/api/v1/users/me',actor=alice,schema='User')['headers']['X-Instance-Id'])
|
||||
check(len(observed)>=2,'less than two API instances observed before failure; verify LB/session affinity')
|
||||
print('Executing explicit '+args.mode+' hook; hook output suppressed to avoid leaking secrets.')
|
||||
# No shell interpolation and no Docker commands in this test.
|
||||
proc = subprocess.Popen([str(script)],env=env,stdout=subprocess.DEVNULL,stderr=subprocess.DEVNULL)
|
||||
started = time.monotonic()
|
||||
successes,failures,after_instances = [],0,[]
|
||||
try:
|
||||
if args.mode=='persistence':
|
||||
try:
|
||||
code = proc.wait(timeout=90)
|
||||
except subprocess.TimeoutExpired:
|
||||
raise Failure('restart hook exceeded 90 seconds') from None
|
||||
check(code==0,'restart hook failed (inspect your script locally)')
|
||||
deadline = time.monotonic()+90
|
||||
while True:
|
||||
try:
|
||||
c.call('GET','/health/ready',schema='Health')
|
||||
c.call('GET','/api/v1/users/me',actor=alice,schema='User')
|
||||
break
|
||||
except Failure:
|
||||
check(time.monotonic()<deadline,'stack/session did not recover within 90 seconds')
|
||||
time.sleep(0.5)
|
||||
else:
|
||||
first_recovery = None
|
||||
key = uuid.uuid4().hex
|
||||
text = 'Запись во время отказа ' + key
|
||||
while time.monotonic()-started < 15:
|
||||
code = proc.poll()
|
||||
check(code is None or code==0,'stop-one hook failed (inspect your script locally)')
|
||||
check(code is not None or time.monotonic()-started<10,'stop-one hook must return within 10 seconds')
|
||||
try:
|
||||
response = c.message(alice,chat,text,key=key)
|
||||
msg = response['json']
|
||||
if code==0:
|
||||
if first_recovery is None:
|
||||
first_recovery = time.monotonic()-started
|
||||
after_instances.append(response['headers']['X-Instance-Id'])
|
||||
successes.append(msg)
|
||||
key = uuid.uuid4().hex
|
||||
text = 'Запись во время отказа ' + key
|
||||
except Failure:
|
||||
failures += 1
|
||||
# Keep SAME key/body after an uncertain POST result.
|
||||
time.sleep(0.1)
|
||||
check(proc.poll()==0,'stop-one hook did not finish successfully')
|
||||
check(first_recovery is not None and first_recovery<=10,'writes did not recover within 10 seconds from hook start')
|
||||
check(len(after_instances)>=5,'too few successful writes after stop')
|
||||
check(target not in after_instances[-5:],'target replica still serves requests; hook must leave it stopped')
|
||||
print(json.dumps({'observed_instances_before':len(observed),'successful_writes':len(successes),'failed_attempts':failures,'first_recovery_seconds':round(first_recovery,3)},ensure_ascii=False))
|
||||
# Existing token, no re-login. Read all confirmations and reject duplicates.
|
||||
messages = history(c,bob,chat)
|
||||
ids = [m['id'] for m in messages]
|
||||
check(len(ids)==len(set(ids)),'duplicated message IDs in history')
|
||||
check(any(m==original for m in messages),'confirmed pre-failure message lost or altered')
|
||||
for msg in successes:
|
||||
check(any(m==msg for m in messages),'confirmed in-flight message lost or altered')
|
||||
# Bodies are unique in this scenario: an uncertain request may exist once, never twice.
|
||||
texts = [m['text'] for m in messages]
|
||||
check(len(texts)==len(set(texts)),'uncertain retry created duplicate message')
|
||||
if attachment:
|
||||
c.call('GET','/api/v1/attachments/'+attachment['id'],actor=bob,schema='Attachment')
|
||||
downloaded = c.call('GET','/api/v1/attachments/'+attachment['id']+'/content',actor=bob)
|
||||
check(downloaded['bytes']==content,'attachment lost or changed across failure')
|
||||
print('PASS: '+args.mode+' preserved confirmed data and existing session. Hook scope needs operator evidence.')
|
||||
finally:
|
||||
if proc.poll() is None:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
proc.wait()
|
||||
|
||||
if __name__=='__main__':
|
||||
run(main)
|
||||
Reference in New Issue
Block a user